hiring cafe scraper
This commit is contained in:
Elshimy Ziad Magdy Taha
parent
8a4ad07e46
commit
a281363b61
31 changed files with 1948 additions and 1648 deletions
14
go.mod
14
go.mod
|
|
@ -3,3 +3,17 @@ module github.com/jobs-scraper
|
|||
go 1.24.0
|
||||
|
||||
toolchain go1.24.7
|
||||
|
||||
require (
|
||||
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d
|
||||
github.com/chromedp/chromedp v0.14.2
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/chromedp/sysutil v1.1.0 // indirect
|
||||
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect
|
||||
github.com/gobwas/httphead v0.1.0 // indirect
|
||||
github.com/gobwas/pool v0.2.1 // indirect
|
||||
github.com/gobwas/ws v1.4.0 // indirect
|
||||
golang.org/x/sys v0.36.0 // indirect
|
||||
)
|
||||
|
|
|
|||
10
go.sum
10
go.sum
|
|
@ -0,0 +1,10 @@
|
|||
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d h1:ZtA1sedVbEW7EW80Iz2GR3Ye6PwbJAJXjv7D74xG6HU=
|
||||
github.com/chromedp/chromedp v0.14.2 h1:r3b/WtwM50RsBZHMUm9fsNhhzRStTHrKdr2zmwbZSzM=
|
||||
github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM=
|
||||
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 h1:iizUGZ9pEquQS5jTGkh4AqeeHCMbfbjeb0zMt0aEFzs=
|
||||
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
|
||||
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
|
||||
github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
|
||||
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
|
||||
1
go.work
1
go.work
|
|
@ -14,6 +14,5 @@ use (
|
|||
./libs/server
|
||||
./services/api
|
||||
./services/scraper-google
|
||||
./services/scraper-hiringcafe
|
||||
./services/scraper-linkedin
|
||||
)
|
||||
|
|
|
|||
14
go.work.sum
14
go.work.sum
|
|
@ -43,6 +43,9 @@ github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J
|
|||
github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
|
||||
github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d/go.mod h1:NItd7aLkcfOA/dcMXvl8p1u+lQqioRMq/SqDp71Pb/k=
|
||||
github.com/chromedp/chromedp v0.14.2/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo=
|
||||
github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8=
|
||||
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
|
||||
github.com/cncf/xds/go v0.0.0-20240723142845-024c85f92f20/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
|
||||
github.com/cockroachdb/cockroach-go/v2 v2.1.1/go.mod h1:7NtUnP6eK+l6k483WSYNrq3Kb23bWV10IRV1TyeSpwM=
|
||||
|
|
@ -57,9 +60,13 @@ github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9
|
|||
github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
|
||||
github.com/fsouza/fake-gcs-server v1.17.0/go.mod h1:D1rTE4YCyHFNa99oyJJ5HyclvN/0uQR+pM/VdlL83bw=
|
||||
github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M=
|
||||
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M=
|
||||
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM=
|
||||
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
|
||||
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
|
||||
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
|
||||
github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/gocql/gocql v0.0.0-20210515062232-b7ef815b4556/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY=
|
||||
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
|
||||
|
|
@ -102,8 +109,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C
|
|||
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
|
||||
github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/ktrysmt/go-bitbucket v0.6.4/go.mod h1:9u0v3hsd2rqCHRIpbir1oP7F58uo5dq19sBYvuMoyQ4=
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
|
||||
github.com/markbates/pkger v0.15.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI=
|
||||
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
|
|
@ -121,6 +128,7 @@ github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8/go.mod h1:86w
|
|||
github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba/go.mod h1:ncO5VaFWh0Nrt+4KT4mOZboaczBZcLuHrG+/sUeP8gI=
|
||||
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
|
||||
github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
|
||||
github.com/pierrec/lz4/v4 v4.1.16/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
|
||||
|
|
@ -156,7 +164,7 @@ golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbht
|
|||
golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
|
||||
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
|
|
@ -169,7 +177,6 @@ golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
|||
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
|
||||
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||
golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||
golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
|
||||
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
|
||||
google.golang.org/api v0.169.0/go.mod h1:gpNOiMA2tZ4mf5R9Iwf4rK/Dcz0fbdIgWYWVoxmsyLg=
|
||||
google.golang.org/api v0.197.0/go.mod h1:AuOuo20GoQ331nq7DquGHlU6d+2wN2fZ8O0ta60nRNw=
|
||||
|
|
@ -183,7 +190,6 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go.
|
|||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
|
||||
google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y=
|
||||
google.golang.org/grpc v1.67.0/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
|
||||
lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
|
||||
modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg=
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ go 1.24.0
|
|||
|
||||
require (
|
||||
github.com/gorilla/mux v1.8.1
|
||||
github.com/gorilla/schema v1.4.1
|
||||
github.com/jobs-scraper/internal/pkg/domain v0.0.0
|
||||
github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0-00010101000000-000000000000
|
||||
github.com/jobs-scraper/internal/pkg/utils v0.0.0
|
||||
|
|
@ -22,7 +23,6 @@ require (
|
|||
github.com/go-openapi/spec v0.20.6 // indirect
|
||||
github.com/go-openapi/swag v0.19.15 // indirect
|
||||
github.com/golang-migrate/migrate/v4 v4.19.0 // indirect
|
||||
github.com/gorilla/schema v1.4.1 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
|
|
|
|||
|
|
@ -17,10 +17,7 @@ export const saveJobs = async (jobs: Job[]) => {
|
|||
job_link: job.jobLink,
|
||||
provider: job.provider,
|
||||
status: job.status,
|
||||
job_timestamp:
|
||||
job.jobPostTime instanceof Date && !isNaN(job.jobPostTime.getTime())
|
||||
? job.jobPostTime
|
||||
: new Date(),
|
||||
job_timestamp: job.jobPostTime,
|
||||
}));
|
||||
|
||||
const result = await sql`
|
||||
|
|
|
|||
|
|
@ -4,8 +4,6 @@ go 1.24.0
|
|||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.10.3
|
||||
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d
|
||||
github.com/chromedp/chromedp v0.14.2
|
||||
github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/levmv/sked v0.2.1
|
||||
|
|
@ -13,17 +11,11 @@ require (
|
|||
|
||||
require (
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/chromedp/sysutil v1.1.0 // indirect
|
||||
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect
|
||||
github.com/gobwas/httphead v0.1.0 // indirect
|
||||
github.com/gobwas/pool v0.2.1 // indirect
|
||||
github.com/gobwas/ws v1.4.0 // indirect
|
||||
github.com/golang-migrate/migrate/v4 v4.19.0 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/lib/pq v1.10.9 // indirect
|
||||
golang.org/x/net v0.44.0 // indirect
|
||||
golang.org/x/sys v0.36.0 // indirect
|
||||
)
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure
|
||||
|
|
|
|||
|
|
@ -2,12 +2,6 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25
|
|||
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
|
||||
github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo=
|
||||
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
|
||||
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d h1:ZtA1sedVbEW7EW80Iz2GR3Ye6PwbJAJXjv7D74xG6HU=
|
||||
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d/go.mod h1:NItd7aLkcfOA/dcMXvl8p1u+lQqioRMq/SqDp71Pb/k=
|
||||
github.com/chromedp/chromedp v0.14.2 h1:r3b/WtwM50RsBZHMUm9fsNhhzRStTHrKdr2zmwbZSzM=
|
||||
github.com/chromedp/chromedp v0.14.2/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo=
|
||||
github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM=
|
||||
github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8=
|
||||
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
|
||||
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
|
|
@ -17,23 +11,13 @@ github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjY
|
|||
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
|
||||
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 h1:iizUGZ9pEquQS5jTGkh4AqeeHCMbfbjeb0zMt0aEFzs=
|
||||
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
|
||||
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
|
||||
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
|
||||
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
|
||||
github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
|
||||
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE=
|
||||
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
||||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
|
||||
github.com/levmv/sked v0.2.1 h1:XMdLkbuajPM5LSzoBvNzalj6oYHkEfeA7/VPvkIHqs0=
|
||||
github.com/levmv/sked v0.2.1/go.mod h1:w9pFUIpZLPu0Jr2+2JqJtaEIHmwXcvjPHWm6pvbqHVQ=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
|
|
@ -42,8 +26,6 @@ github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
|
|||
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
|
|
@ -53,7 +35,6 @@ go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
|
|||
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
|
||||
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
|
||||
golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
|
||||
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
|
|
|
|||
|
|
@ -19,11 +19,11 @@ func Work(mainCtx context.Context, b *browser.Browser) func(page int, stream cha
|
|||
}
|
||||
defer tab.Close()
|
||||
|
||||
// Random delay before starting to look more human
|
||||
browser.RandomDelay(1000, 3000)
|
||||
|
||||
// query := "site:boards.greenhouse.io (Software OR Backend OR Full-Stack) Engineer inurl:gh_jid"
|
||||
query := `site:lever.co (Software OR Backend OR Full-Stack) Engineer lang:en`
|
||||
// query := `site:lever.co (Software OR Backend OR Full-Stack) Engineer lang:en`
|
||||
query := `site:workable.com (Software OR Backend OR Full-Stack) Engineer lang:en`
|
||||
// url := utils.BuildUrl(query, page)
|
||||
err = tab.Navigate("https://linkedin.com")
|
||||
|
||||
|
|
|
|||
12
services/scraper-hiringcafe/.env
Normal file
12
services/scraper-hiringcafe/.env
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
DB_HOST=localhost
|
||||
DB_PORT=5432
|
||||
DB_USER=postgres
|
||||
DB_PASSWORD=password
|
||||
DB_NAME=linkedin_jobs
|
||||
DB_SSLMODE=disable
|
||||
|
||||
SERVER_PORT=8080
|
||||
SERVER_HOST=localhost
|
||||
|
||||
CV_AI_MODEL=your_ai_model
|
||||
OPENROUTER_API_KEY=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
|
||||
|
|
@ -1,245 +0,0 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
BaseURL = "https://hiring.cafe"
|
||||
SearchURL = BaseURL + "/api/search-jobs"
|
||||
UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
PageSize = 20
|
||||
)
|
||||
|
||||
// Client handles API requests to hiring.cafe
|
||||
type Client struct {
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// NewClient creates a new API client
|
||||
func NewClient() *Client {
|
||||
return &Client{
|
||||
httpClient: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// SearchRequest represents the search request payload (based on RSSHub implementation)
|
||||
type SearchRequest struct {
|
||||
Size int `json:"size"`
|
||||
Page int `json:"page"`
|
||||
SearchState SearchState `json:"searchState"`
|
||||
}
|
||||
|
||||
// SearchState contains search parameters
|
||||
type SearchState struct {
|
||||
SearchQuery string `json:"searchQuery"`
|
||||
SortBy string `json:"sortBy"` // date, default, compensation_desc, experience_asc
|
||||
}
|
||||
|
||||
// SearchResponse represents the API response
|
||||
type SearchResponse struct {
|
||||
Results []JobResult `json:"results"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
|
||||
// JobResult represents a job result from the API
|
||||
type JobResult struct {
|
||||
ID string `json:"id"`
|
||||
ApplyURL string `json:"apply_url"`
|
||||
JobInformation JobInformation `json:"job_information"`
|
||||
ProcessedJobData ProcessedJobData `json:"v5_processed_job_data"`
|
||||
}
|
||||
|
||||
// JobInformation contains basic job info
|
||||
type JobInformation struct {
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
}
|
||||
|
||||
// ProcessedJobData contains processed job details
|
||||
type ProcessedJobData struct {
|
||||
CompanyName string `json:"company_name"`
|
||||
IsCompensationTransparent bool `json:"is_compensation_transparent"`
|
||||
YearlyMinCompensation *float64 `json:"yearly_min_compensation"`
|
||||
YearlyMaxCompensation *float64 `json:"yearly_max_compensation"`
|
||||
WorkplaceType string `json:"workplace_type"`
|
||||
RequirementsSummary string `json:"requirements_summary"`
|
||||
JobCategory string `json:"job_category"`
|
||||
RoleActivities []string `json:"role_activities"`
|
||||
FormattedWorkplaceLocation string `json:"formatted_workplace_location"`
|
||||
EstimatedPublishDateMillis string `json:"estimated_publish_date_millis"`
|
||||
}
|
||||
|
||||
// Job is a simplified job structure for external use
|
||||
type Job struct {
|
||||
ID string
|
||||
ApplyURL string
|
||||
Title string
|
||||
Description string
|
||||
Company string
|
||||
CompanyURL string
|
||||
Location string
|
||||
Remote bool
|
||||
Salary string
|
||||
PostedAt time.Time
|
||||
DescriptionClean string
|
||||
}
|
||||
|
||||
// SearchFilters contains user-configurable search filters
|
||||
type SearchFilters struct {
|
||||
Query string
|
||||
Country string
|
||||
Remote bool
|
||||
Hybrid bool
|
||||
Onsite bool
|
||||
FullTime bool
|
||||
PartTime bool
|
||||
Contract bool
|
||||
Internship bool
|
||||
EntryLevel bool
|
||||
MidLevel bool
|
||||
SeniorLevel bool
|
||||
DateRangeDays int
|
||||
MaxPages int
|
||||
}
|
||||
|
||||
// DefaultFilters returns default search filters
|
||||
func DefaultFilters() SearchFilters {
|
||||
return SearchFilters{
|
||||
Query: "",
|
||||
Country: "United States",
|
||||
Remote: true,
|
||||
Hybrid: true,
|
||||
Onsite: true,
|
||||
FullTime: true,
|
||||
PartTime: false,
|
||||
Contract: false,
|
||||
Internship: false,
|
||||
EntryLevel: true,
|
||||
MidLevel: true,
|
||||
SeniorLevel: false,
|
||||
DateRangeDays: 30,
|
||||
MaxPages: 10,
|
||||
}
|
||||
}
|
||||
|
||||
// Search performs a job search with the given filters
|
||||
func (c *Client) Search(filters SearchFilters) ([]Job, error) {
|
||||
var allJobs []Job
|
||||
page := 1
|
||||
|
||||
for {
|
||||
jobs, totalCount, err := c.searchPage(filters, page)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to search page %d: %w", page, err)
|
||||
}
|
||||
|
||||
allJobs = append(allJobs, jobs...)
|
||||
|
||||
// Check if we've fetched all jobs or reached max pages
|
||||
if len(allJobs) >= totalCount || page >= filters.MaxPages || len(jobs) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
page++
|
||||
// Rate limiting - be nice to the API
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
|
||||
return allJobs, nil
|
||||
}
|
||||
|
||||
// searchPage performs a single page search using the correct API format
|
||||
func (c *Client) searchPage(filters SearchFilters, page int) ([]Job, int, error) {
|
||||
reqBody := SearchRequest{
|
||||
Size: PageSize,
|
||||
Page: page - 1, // API uses 0-based indexing
|
||||
SearchState: SearchState{
|
||||
SearchQuery: filters.Query,
|
||||
SortBy: "date",
|
||||
},
|
||||
}
|
||||
|
||||
jsonBody, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("failed to marshal request: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("Sending request to %s with body: %s", SearchURL, string(jsonBody))
|
||||
|
||||
req, err := http.NewRequest("POST", SearchURL, bytes.NewBuffer(jsonBody))
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", UserAgent)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Origin", BaseURL)
|
||||
req.Header.Set("Referer", BaseURL+"/")
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("failed to send request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("failed to read response: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, 0, fmt.Errorf("API returned status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
var searchResp SearchResponse
|
||||
if err := json.Unmarshal(body, &searchResp); err != nil {
|
||||
return nil, 0, fmt.Errorf("failed to unmarshal response: %w", err)
|
||||
}
|
||||
|
||||
// Convert JobResult to Job
|
||||
var jobs []Job
|
||||
for _, result := range searchResp.Results {
|
||||
job := Job{
|
||||
ID: result.ID,
|
||||
ApplyURL: result.ApplyURL,
|
||||
Title: result.JobInformation.Title,
|
||||
Description: result.JobInformation.Description,
|
||||
Company: result.ProcessedJobData.CompanyName,
|
||||
Location: result.ProcessedJobData.FormattedWorkplaceLocation,
|
||||
Remote: result.ProcessedJobData.WorkplaceType == "remote",
|
||||
DescriptionClean: result.JobInformation.Description,
|
||||
}
|
||||
|
||||
// Parse salary if available
|
||||
if result.ProcessedJobData.IsCompensationTransparent &&
|
||||
result.ProcessedJobData.YearlyMinCompensation != nil &&
|
||||
result.ProcessedJobData.YearlyMaxCompensation != nil {
|
||||
job.Salary = fmt.Sprintf("$%.0f - $%.0f",
|
||||
*result.ProcessedJobData.YearlyMinCompensation,
|
||||
*result.ProcessedJobData.YearlyMaxCompensation)
|
||||
}
|
||||
|
||||
// Parse posted date
|
||||
if result.ProcessedJobData.EstimatedPublishDateMillis != "" {
|
||||
// Try to parse as milliseconds timestamp
|
||||
var millis int64
|
||||
if _, err := fmt.Sscanf(result.ProcessedJobData.EstimatedPublishDateMillis, "%d", &millis); err == nil {
|
||||
job.PostedAt = time.UnixMilli(millis)
|
||||
}
|
||||
}
|
||||
|
||||
jobs = append(jobs, job)
|
||||
}
|
||||
|
||||
return jobs, searchResp.Total, nil
|
||||
}
|
||||
|
|
@ -1,583 +0,0 @@
|
|||
package browser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/rand"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/chromedp/cdproto/cdp"
|
||||
"github.com/chromedp/cdproto/page"
|
||||
"github.com/chromedp/chromedp"
|
||||
)
|
||||
|
||||
// Config holds browser configuration options
|
||||
type Config struct {
|
||||
Headless bool
|
||||
Timeout time.Duration
|
||||
UserDataDir string
|
||||
DebugPort int
|
||||
UseRealChrome bool // Connect to real Chrome instead of launching automated one
|
||||
ConnectToExisting bool // Connect to already running Chrome (user must start it manually)
|
||||
}
|
||||
|
||||
// DefaultConfig returns default browser configuration
|
||||
func DefaultConfig() Config {
|
||||
return Config{
|
||||
Headless: false,
|
||||
Timeout: 10 * time.Minute,
|
||||
UserDataDir: filepath.Join(os.TempDir(), "hiringcafe-chrome-profile"),
|
||||
DebugPort: 9224, // Use different port to avoid conflicts
|
||||
UseRealChrome: true,
|
||||
ConnectToExisting: false,
|
||||
}
|
||||
}
|
||||
|
||||
// Browser wraps chromedp browser context with anti-detection features
|
||||
type Browser struct {
|
||||
allocCtx context.Context
|
||||
allocCancel context.CancelFunc
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
config Config
|
||||
chromeCmd *exec.Cmd
|
||||
}
|
||||
|
||||
// chromeResponse holds the Chrome DevTools Protocol response
|
||||
type chromeResponse struct {
|
||||
WebSocketDebuggerURL string `json:"webSocketDebuggerUrl"`
|
||||
}
|
||||
|
||||
// ErrCaptchaDetected is returned when bot detection/CAPTCHA is detected
|
||||
var ErrCaptchaDetected = fmt.Errorf("captcha or bot detection detected - terminating")
|
||||
|
||||
// New creates a new browser instance with anti-detection measures
|
||||
func New(parentCtx context.Context, config Config) (*Browser, error) {
|
||||
// Always use the automated real Chrome approach now
|
||||
return newRealChromeAutomated(parentCtx, config)
|
||||
}
|
||||
|
||||
// newRealChromeAutomated launches Chrome like a normal user would and connects to it
|
||||
func newRealChromeAutomated(parentCtx context.Context, config Config) (*Browser, error) {
|
||||
b := &Browser{config: config}
|
||||
|
||||
// Check if Chrome is already running on the debug port
|
||||
conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", config.DebugPort), time.Second)
|
||||
if err == nil {
|
||||
conn.Close()
|
||||
log.Printf("Chrome already running on port %d, connecting to it...", config.DebugPort)
|
||||
} else {
|
||||
// Launch Chrome with remote debugging
|
||||
if err := b.launchChromeClean(); err != nil {
|
||||
return nil, fmt.Errorf("failed to launch Chrome: %w", err)
|
||||
}
|
||||
|
||||
// Wait for Chrome to be ready
|
||||
if err := b.waitForChrome(); err != nil {
|
||||
b.Close()
|
||||
return nil, fmt.Errorf("chrome not ready: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Get WebSocket URL
|
||||
wsURL, err := b.getWebSocketURL()
|
||||
if err != nil {
|
||||
b.Close()
|
||||
return nil, fmt.Errorf("failed to get WebSocket URL: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("Connected to Chrome at %s", wsURL)
|
||||
|
||||
// Connect to Chrome via WebSocket
|
||||
allocCtx, allocCancel := chromedp.NewRemoteAllocator(parentCtx, wsURL)
|
||||
ctx, cancel := chromedp.NewContext(allocCtx)
|
||||
|
||||
// Set timeout
|
||||
ctx, timeoutCancel := context.WithTimeout(ctx, config.Timeout)
|
||||
|
||||
b.allocCtx = allocCtx
|
||||
b.allocCancel = allocCancel
|
||||
b.ctx = ctx
|
||||
b.cancel = func() {
|
||||
timeoutCancel()
|
||||
cancel()
|
||||
}
|
||||
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// launchChromeClean launches Chrome as cleanly as possible (like a normal user)
|
||||
func (b *Browser) launchChromeClean() error {
|
||||
chromePath := findChromePath()
|
||||
if chromePath == "" {
|
||||
return fmt.Errorf("chrome not found")
|
||||
}
|
||||
|
||||
// Ensure user data directory exists
|
||||
if err := os.MkdirAll(b.config.UserDataDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create user data dir: %w", err)
|
||||
}
|
||||
|
||||
// Minimal args - just what's needed for remote debugging
|
||||
// Avoid any automation-related flags that might be detected
|
||||
args := []string{
|
||||
fmt.Sprintf("--remote-debugging-port=%d", b.config.DebugPort),
|
||||
fmt.Sprintf("--user-data-dir=%s", b.config.UserDataDir),
|
||||
"--remote-allow-origins=*",
|
||||
"--no-first-run",
|
||||
"--no-default-browser-check",
|
||||
"--start-maximized",
|
||||
}
|
||||
|
||||
log.Printf("Launching Chrome: %s", chromePath)
|
||||
b.chromeCmd = exec.Command(chromePath, args...)
|
||||
b.chromeCmd.Stdout = nil
|
||||
b.chromeCmd.Stderr = nil
|
||||
|
||||
if err := b.chromeCmd.Start(); err != nil {
|
||||
return fmt.Errorf("failed to start Chrome: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("Chrome launched with PID %d on port %d", b.chromeCmd.Process.Pid, b.config.DebugPort)
|
||||
return nil
|
||||
}
|
||||
|
||||
// newRealChrome launches a real Chrome browser and connects via DevTools Protocol
|
||||
func newRealChrome(parentCtx context.Context, config Config) (*Browser, error) {
|
||||
b := &Browser{config: config}
|
||||
|
||||
// Launch Chrome with remote debugging
|
||||
if err := b.launchChrome(); err != nil {
|
||||
return nil, fmt.Errorf("failed to launch Chrome: %w", err)
|
||||
}
|
||||
|
||||
// Wait for Chrome to be ready
|
||||
if err := b.waitForChrome(); err != nil {
|
||||
b.Close()
|
||||
return nil, fmt.Errorf("Chrome not ready: %w", err)
|
||||
}
|
||||
|
||||
// Get WebSocket URL
|
||||
wsURL, err := b.getWebSocketURL()
|
||||
if err != nil {
|
||||
b.Close()
|
||||
return nil, fmt.Errorf("failed to get WebSocket URL: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("Connecting to Chrome at %s", wsURL)
|
||||
|
||||
// Connect to Chrome via WebSocket
|
||||
allocCtx, allocCancel := chromedp.NewRemoteAllocator(parentCtx, wsURL)
|
||||
ctx, cancel := chromedp.NewContext(allocCtx)
|
||||
|
||||
// Set timeout
|
||||
ctx, timeoutCancel := context.WithTimeout(ctx, config.Timeout)
|
||||
|
||||
b.allocCtx = allocCtx
|
||||
b.allocCancel = allocCancel
|
||||
b.ctx = ctx
|
||||
b.cancel = func() {
|
||||
timeoutCancel()
|
||||
cancel()
|
||||
}
|
||||
|
||||
// Inject anti-detection scripts
|
||||
if err := b.injectAntiDetection(); err != nil {
|
||||
log.Printf("Warning: could not inject anti-detection: %v", err)
|
||||
}
|
||||
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// launchChrome launches Chrome with remote debugging enabled
|
||||
func (b *Browser) launchChrome() error {
|
||||
chromePath := findChromePath()
|
||||
if chromePath == "" {
|
||||
return fmt.Errorf("Chrome not found")
|
||||
}
|
||||
|
||||
// Ensure user data directory exists
|
||||
if err := os.MkdirAll(b.config.UserDataDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create user data dir: %w", err)
|
||||
}
|
||||
|
||||
args := []string{
|
||||
fmt.Sprintf("--remote-debugging-port=%d", b.config.DebugPort),
|
||||
fmt.Sprintf("--user-data-dir=%s", b.config.UserDataDir),
|
||||
"--remote-allow-origins=*",
|
||||
"--no-first-run",
|
||||
"--no-default-browser-check",
|
||||
"--disable-background-networking",
|
||||
"--disable-client-side-phishing-detection",
|
||||
"--disable-default-apps",
|
||||
"--disable-hang-monitor",
|
||||
"--disable-popup-blocking",
|
||||
"--disable-prompt-on-repost",
|
||||
"--disable-sync",
|
||||
"--disable-translate",
|
||||
"--metrics-recording-only",
|
||||
"--safebrowsing-disable-auto-update",
|
||||
"--password-store=basic",
|
||||
"--use-mock-keychain",
|
||||
}
|
||||
|
||||
log.Printf("Launching Chrome: %s", chromePath)
|
||||
b.chromeCmd = exec.Command(chromePath, args...)
|
||||
b.chromeCmd.Stdout = nil
|
||||
b.chromeCmd.Stderr = nil
|
||||
|
||||
if err := b.chromeCmd.Start(); err != nil {
|
||||
return fmt.Errorf("failed to start Chrome: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("Chrome launched with PID %d on port %d", b.chromeCmd.Process.Pid, b.config.DebugPort)
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitForChrome waits for Chrome to be ready to accept connections
|
||||
func (b *Browser) waitForChrome() error {
|
||||
timeout := time.After(30 * time.Second)
|
||||
ticker := time.NewTicker(500 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-timeout:
|
||||
return fmt.Errorf("timeout waiting for Chrome to start")
|
||||
case <-ticker.C:
|
||||
conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", b.config.DebugPort), time.Second)
|
||||
if err == nil {
|
||||
conn.Close()
|
||||
time.Sleep(500 * time.Millisecond) // Give Chrome a moment to fully initialize
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// getWebSocketURL gets the WebSocket debugger URL from Chrome
|
||||
func (b *Browser) getWebSocketURL() (string, error) {
|
||||
resp, err := http.Get(fmt.Sprintf("http://localhost:%d/json/version", b.config.DebugPort))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var result chromeResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return result.WebSocketDebuggerURL, nil
|
||||
}
|
||||
|
||||
// findChromePath finds the Chrome executable path
|
||||
func findChromePath() string {
|
||||
var paths []string
|
||||
|
||||
switch runtime.GOOS {
|
||||
case "darwin":
|
||||
paths = []string{
|
||||
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
||||
"/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
|
||||
"/Applications/Chromium.app/Contents/MacOS/Chromium",
|
||||
}
|
||||
case "linux":
|
||||
paths = []string{
|
||||
"/usr/bin/google-chrome",
|
||||
"/usr/bin/google-chrome-stable",
|
||||
"/usr/bin/chromium",
|
||||
"/usr/bin/chromium-browser",
|
||||
}
|
||||
case "windows":
|
||||
paths = []string{
|
||||
os.Getenv("LOCALAPPDATA") + "\\Google\\Chrome\\Application\\chrome.exe",
|
||||
os.Getenv("PROGRAMFILES") + "\\Google\\Chrome\\Application\\chrome.exe",
|
||||
os.Getenv("PROGRAMFILES(X86)") + "\\Google\\Chrome\\Application\\chrome.exe",
|
||||
}
|
||||
}
|
||||
|
||||
for _, p := range paths {
|
||||
if _, err := os.Stat(p); err == nil {
|
||||
return p
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// newAutomatedChrome creates a browser using chromedp's built-in launcher (fallback)
|
||||
func newAutomatedChrome(parentCtx context.Context, config Config) (*Browser, error) {
|
||||
opts := []chromedp.ExecAllocatorOption{
|
||||
chromedp.NoFirstRun,
|
||||
chromedp.NoDefaultBrowserCheck,
|
||||
chromedp.DisableGPU,
|
||||
chromedp.NoSandbox,
|
||||
chromedp.Flag("disable-blink-features", "AutomationControlled"),
|
||||
chromedp.Flag("disable-infobars", true),
|
||||
chromedp.Flag("disable-dev-shm-usage", true),
|
||||
chromedp.Flag("window-size", "1920,1080"),
|
||||
chromedp.UserAgent(getRandomUserAgent()),
|
||||
}
|
||||
|
||||
if config.Headless {
|
||||
opts = append(opts, chromedp.Flag("headless", "new"))
|
||||
}
|
||||
|
||||
if config.UserDataDir != "" {
|
||||
opts = append(opts, chromedp.UserDataDir(config.UserDataDir))
|
||||
}
|
||||
|
||||
allocCtx, allocCancel := chromedp.NewExecAllocator(parentCtx, opts...)
|
||||
ctx, cancel := chromedp.NewContext(allocCtx, chromedp.WithLogf(log.Printf))
|
||||
ctx, timeoutCancel := context.WithTimeout(ctx, config.Timeout)
|
||||
|
||||
b := &Browser{
|
||||
allocCtx: allocCtx,
|
||||
allocCancel: allocCancel,
|
||||
ctx: ctx,
|
||||
cancel: func() {
|
||||
timeoutCancel()
|
||||
cancel()
|
||||
},
|
||||
config: config,
|
||||
}
|
||||
|
||||
if err := b.injectAntiDetection(); err != nil {
|
||||
b.Close()
|
||||
return nil, fmt.Errorf("failed to inject anti-detection: %w", err)
|
||||
}
|
||||
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// Context returns the browser context
|
||||
func (b *Browser) Context() context.Context {
|
||||
return b.ctx
|
||||
}
|
||||
|
||||
// Close closes the browser and kills Chrome process if we launched it
|
||||
func (b *Browser) Close() {
|
||||
if b.cancel != nil {
|
||||
b.cancel()
|
||||
}
|
||||
if b.allocCancel != nil {
|
||||
b.allocCancel()
|
||||
}
|
||||
// Kill Chrome process if we launched it
|
||||
if b.chromeCmd != nil && b.chromeCmd.Process != nil {
|
||||
log.Println("Killing Chrome process...")
|
||||
b.chromeCmd.Process.Kill()
|
||||
}
|
||||
}
|
||||
|
||||
// injectAntiDetection injects JavaScript to bypass bot detection
|
||||
func (b *Browser) injectAntiDetection() error {
|
||||
// JavaScript to override navigator properties and hide automation
|
||||
antiDetectionScript := `
|
||||
// Override webdriver property
|
||||
Object.defineProperty(navigator, 'webdriver', {
|
||||
get: () => undefined,
|
||||
});
|
||||
|
||||
// Override plugins
|
||||
Object.defineProperty(navigator, 'plugins', {
|
||||
get: () => [1, 2, 3, 4, 5],
|
||||
});
|
||||
|
||||
// Override languages
|
||||
Object.defineProperty(navigator, 'languages', {
|
||||
get: () => ['en-US', 'en'],
|
||||
});
|
||||
|
||||
// Override platform
|
||||
Object.defineProperty(navigator, 'platform', {
|
||||
get: () => 'MacIntel',
|
||||
});
|
||||
|
||||
// Override hardware concurrency
|
||||
Object.defineProperty(navigator, 'hardwareConcurrency', {
|
||||
get: () => 8,
|
||||
});
|
||||
|
||||
// Override device memory
|
||||
Object.defineProperty(navigator, 'deviceMemory', {
|
||||
get: () => 8,
|
||||
});
|
||||
|
||||
// Override permissions query
|
||||
const originalQuery = window.navigator.permissions.query;
|
||||
window.navigator.permissions.query = (parameters) => (
|
||||
parameters.name === 'notifications' ?
|
||||
Promise.resolve({ state: Notification.permission }) :
|
||||
originalQuery(parameters)
|
||||
);
|
||||
|
||||
// Override chrome runtime
|
||||
window.chrome = {
|
||||
runtime: {},
|
||||
};
|
||||
|
||||
// Override console.debug to prevent detection
|
||||
window.console.debug = () => {};
|
||||
|
||||
// Add mouse movement simulation capability
|
||||
window.simulateMouseMovement = function() {
|
||||
const event = new MouseEvent('mousemove', {
|
||||
bubbles: true,
|
||||
cancelable: true,
|
||||
clientX: Math.random() * window.innerWidth,
|
||||
clientY: Math.random() * window.innerHeight
|
||||
});
|
||||
document.dispatchEvent(event);
|
||||
};
|
||||
`
|
||||
|
||||
return chromedp.Run(b.ctx,
|
||||
chromedp.ActionFunc(func(ctx context.Context) error {
|
||||
_, err := page.AddScriptToEvaluateOnNewDocument(antiDetectionScript).Do(ctx)
|
||||
return err
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
// Navigate navigates to a URL and checks for CAPTCHA/bot detection
|
||||
func (b *Browser) Navigate(url string) error {
|
||||
if err := chromedp.Run(b.ctx,
|
||||
chromedp.Navigate(url),
|
||||
chromedp.WaitVisible("body", chromedp.ByQuery),
|
||||
chromedp.Sleep(2*time.Second),
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check for CAPTCHA/bot detection
|
||||
if detected, err := b.CheckForCaptcha(); err != nil {
|
||||
return err
|
||||
} else if detected {
|
||||
return ErrCaptchaDetected
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// CheckForCaptcha checks if the page shows CAPTCHA or bot detection
|
||||
func (b *Browser) CheckForCaptcha() (bool, error) {
|
||||
var pageContent string
|
||||
var pageTitle string
|
||||
|
||||
if err := chromedp.Run(b.ctx,
|
||||
chromedp.Title(&pageTitle),
|
||||
chromedp.Evaluate(`document.body.innerText`, &pageContent),
|
||||
); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// Check for common CAPTCHA/bot detection indicators
|
||||
captchaIndicators := []string{
|
||||
"Failed to verify your browser",
|
||||
"Vercel Security Checkpoint",
|
||||
"Please verify you are a human",
|
||||
"Access denied",
|
||||
"Checking your browser",
|
||||
"Please complete the security check",
|
||||
"captcha",
|
||||
"CAPTCHA",
|
||||
"robot",
|
||||
"bot detection",
|
||||
"security challenge",
|
||||
"Code 10",
|
||||
"Code 21",
|
||||
"Code 22",
|
||||
}
|
||||
|
||||
contentLower := strings.ToLower(pageContent)
|
||||
titleLower := strings.ToLower(pageTitle)
|
||||
|
||||
for _, indicator := range captchaIndicators {
|
||||
indicatorLower := strings.ToLower(indicator)
|
||||
if strings.Contains(contentLower, indicatorLower) || strings.Contains(titleLower, indicatorLower) {
|
||||
log.Printf("CAPTCHA/Bot detection detected: found '%s'", indicator)
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// WaitAndClick waits for an element and clicks it with human-like delay
|
||||
func (b *Browser) WaitAndClick(selector string) error {
|
||||
RandomDelay(500, 1500)
|
||||
return chromedp.Run(b.ctx,
|
||||
chromedp.WaitVisible(selector, chromedp.ByQuery),
|
||||
chromedp.Click(selector, chromedp.ByQuery),
|
||||
)
|
||||
}
|
||||
|
||||
// Type types text with human-like delays between keystrokes
|
||||
func (b *Browser) Type(selector, text string) error {
|
||||
RandomDelay(300, 800)
|
||||
return chromedp.Run(b.ctx,
|
||||
chromedp.WaitVisible(selector, chromedp.ByQuery),
|
||||
chromedp.Click(selector, chromedp.ByQuery),
|
||||
chromedp.Sleep(200*time.Millisecond),
|
||||
chromedp.SendKeys(selector, text, chromedp.ByQuery),
|
||||
)
|
||||
}
|
||||
|
||||
// Evaluate evaluates JavaScript and returns the result
|
||||
func (b *Browser) Evaluate(script string, result interface{}) error {
|
||||
return chromedp.Run(b.ctx,
|
||||
chromedp.Evaluate(script, result),
|
||||
)
|
||||
}
|
||||
|
||||
// Sleep pauses execution for the specified duration
|
||||
func (b *Browser) Sleep(d time.Duration) error {
|
||||
return chromedp.Run(b.ctx, chromedp.Sleep(d))
|
||||
}
|
||||
|
||||
// Nodes returns nodes matching the selector
|
||||
func (b *Browser) Nodes(selector string) ([]*cdp.Node, error) {
|
||||
var nodes []*cdp.Node
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.Nodes(selector, &nodes, chromedp.ByQueryAll),
|
||||
)
|
||||
return nodes, err
|
||||
}
|
||||
|
||||
// SimulateHumanBehavior performs random mouse movements and scrolls
|
||||
func (b *Browser) SimulateHumanBehavior() error {
|
||||
return chromedp.Run(b.ctx,
|
||||
chromedp.Evaluate(`window.simulateMouseMovement()`, nil),
|
||||
chromedp.Sleep(time.Duration(rand.Intn(500)+200)*time.Millisecond),
|
||||
chromedp.Evaluate(`window.scrollBy(0, Math.random() * 100)`, nil),
|
||||
)
|
||||
}
|
||||
|
||||
// RandomDelay waits for a random duration between min and max milliseconds
|
||||
func RandomDelay(minMs, maxMs int) {
|
||||
delay := time.Duration(rand.Intn(maxMs-minMs)+minMs) * time.Millisecond
|
||||
time.Sleep(delay)
|
||||
}
|
||||
|
||||
// getRandomUserAgent returns a random modern user agent
|
||||
func getRandomUserAgent() string {
|
||||
userAgents := []string{
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
|
||||
}
|
||||
return userAgents[rand.Intn(len(userAgents))]
|
||||
}
|
||||
19
services/scraper-hiringcafe/db/db.ts
Normal file
19
services/scraper-hiringcafe/db/db.ts
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
import postgres from "postgres";
|
||||
import dotenv from "dotenv";
|
||||
|
||||
if (dotenv.config({ path: ".local.env" }).error) {
|
||||
console.log("No .local.env file found, trying .env");
|
||||
if (dotenv.config().error) {
|
||||
console.log("No .env file found, using system environment variables");
|
||||
}
|
||||
}
|
||||
|
||||
const sql = postgres({
|
||||
host: process.env.DB_HOST,
|
||||
port: Number(process.env.DB_PORT) || 5432,
|
||||
database: process.env.DB_NAME,
|
||||
username: process.env.DB_USER,
|
||||
password: process.env.DB_PASSWORD,
|
||||
});
|
||||
|
||||
export default sql;
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
module github.com/jobs-scraper/services/scraper-hiringcafe
|
||||
|
||||
go 1.24.0
|
||||
|
||||
require (
|
||||
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d
|
||||
github.com/chromedp/chromedp v0.14.2
|
||||
github.com/jobs-scraper/internal/pkg/domain v0.0.0
|
||||
github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0
|
||||
github.com/jobs-scraper/libs/repo v0.0.0
|
||||
github.com/joho/godotenv v1.5.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/chromedp/sysutil v1.1.0 // indirect
|
||||
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect
|
||||
github.com/gobwas/httphead v0.1.0 // indirect
|
||||
github.com/gobwas/pool v0.2.1 // indirect
|
||||
github.com/gobwas/ws v1.4.0 // indirect
|
||||
github.com/golang-migrate/migrate/v4 v4.19.0 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/lib/pq v1.10.9 // indirect
|
||||
golang.org/x/sys v0.36.0 // indirect
|
||||
)
|
||||
|
||||
replace (
|
||||
github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain
|
||||
github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure
|
||||
github.com/jobs-scraper/libs/repo => ../../libs/repo
|
||||
)
|
||||
|
|
@ -1,88 +0,0 @@
|
|||
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
|
||||
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
|
||||
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
|
||||
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
|
||||
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d h1:ZtA1sedVbEW7EW80Iz2GR3Ye6PwbJAJXjv7D74xG6HU=
|
||||
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d/go.mod h1:NItd7aLkcfOA/dcMXvl8p1u+lQqioRMq/SqDp71Pb/k=
|
||||
github.com/chromedp/chromedp v0.14.2 h1:r3b/WtwM50RsBZHMUm9fsNhhzRStTHrKdr2zmwbZSzM=
|
||||
github.com/chromedp/chromedp v0.14.2/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo=
|
||||
github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM=
|
||||
github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8=
|
||||
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
|
||||
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
|
||||
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
|
||||
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4=
|
||||
github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU=
|
||||
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||
github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI=
|
||||
github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
|
||||
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
|
||||
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
|
||||
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 h1:iizUGZ9pEquQS5jTGkh4AqeeHCMbfbjeb0zMt0aEFzs=
|
||||
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
|
||||
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
|
||||
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
|
||||
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
|
||||
github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
|
||||
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE=
|
||||
github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
||||
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
|
||||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
||||
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
|
||||
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
|
||||
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
|
||||
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
|
||||
github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8=
|
||||
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
|
||||
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
|
||||
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
|
||||
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
|
||||
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
|
||||
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
|
||||
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
189
services/scraper-hiringcafe/index.ts
Normal file
189
services/scraper-hiringcafe/index.ts
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
import puppeteer, { Browser, Page } from "puppeteer-core";
|
||||
import { ChildProcess } from "child_process";
|
||||
import * as dotenv from "dotenv";
|
||||
import {
|
||||
ChromeVersionResponse,
|
||||
launchChromeWithDebugging,
|
||||
randomDelay,
|
||||
waitForChromeReady,
|
||||
} from "./utils/browser";
|
||||
import { BrowserService } from "./services/browserService";
|
||||
import { saveJob } from "./queries/save-jobs";
|
||||
import { saveJobDescription } from "./queries/save-job-description";
|
||||
import { Job } from "./types/job";
|
||||
|
||||
if (dotenv.config({ path: ".local.env" }).error) {
|
||||
console.log("No .local.env file found, trying .env");
|
||||
if (dotenv.config().error) {
|
||||
console.log("No .env file found, using system environment variables");
|
||||
}
|
||||
}
|
||||
|
||||
let globalBrowser: Browser | null = null;
|
||||
let chrome: ChildProcess;
|
||||
|
||||
async function cleanup() {
|
||||
console.log("Starting cleanup process...");
|
||||
|
||||
if (globalBrowser) {
|
||||
console.log("Closing browser on app shutdown");
|
||||
await globalBrowser.close();
|
||||
globalBrowser = null;
|
||||
}
|
||||
|
||||
if (chrome) {
|
||||
chrome.kill("SIGINT");
|
||||
}
|
||||
|
||||
console.log("Cleanup completed");
|
||||
}
|
||||
|
||||
async function initializeBrowser(): Promise<{ browser: Browser; page: Page }> {
|
||||
chrome = launchChromeWithDebugging();
|
||||
await waitForChromeReady();
|
||||
|
||||
const response = await fetch("http://localhost:9222/json/version", {
|
||||
headers: {
|
||||
Origin: "",
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error("Failed to connect to Chrome debugging port");
|
||||
}
|
||||
|
||||
const chromeInstance = (await response.json()) as ChromeVersionResponse;
|
||||
const { webSocketDebuggerUrl } = chromeInstance;
|
||||
|
||||
const browser = await puppeteer.connect({
|
||||
browserWSEndpoint: webSocketDebuggerUrl,
|
||||
defaultViewport: null,
|
||||
});
|
||||
|
||||
globalBrowser = browser;
|
||||
console.log("Connected to browser successfully");
|
||||
|
||||
let pages = await browser.pages();
|
||||
console.log(`Found ${pages.length} pages`);
|
||||
|
||||
let existingcookies = await browser.cookies();
|
||||
await browser.deleteCookie(...existingcookies);
|
||||
|
||||
let page: Page;
|
||||
if (pages.length === 0) {
|
||||
console.log("No pages found, creating a new page");
|
||||
page = await browser.newPage();
|
||||
} else {
|
||||
page = pages[0];
|
||||
}
|
||||
|
||||
await page.setUserAgent(
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
);
|
||||
|
||||
return { browser, page };
|
||||
}
|
||||
|
||||
async function scrapeHiringCafeJobs(): Promise<void> {
|
||||
try {
|
||||
const { page } = await initializeBrowser();
|
||||
const browserService = new BrowserService(page);
|
||||
|
||||
const searchUrl = `https://hiring.cafe/?searchState=%7B%22defaultToUserLocation%22%3Afalse%2C%22searchQuery%22%3A%22software%22%2C%22dateFetchedPastNDays%22%3A2%7D`;
|
||||
|
||||
await page.goto(searchUrl, {
|
||||
waitUntil: ["networkidle2", "domcontentloaded"],
|
||||
timeout: 60000,
|
||||
});
|
||||
|
||||
const searchElement = await page.waitForSelector("#query-search-v4", {
|
||||
timeout: 60000,
|
||||
});
|
||||
|
||||
if (searchElement != null) {
|
||||
console.log("found search element, continue");
|
||||
} else {
|
||||
console.log("could not find search element, aborting");
|
||||
return;
|
||||
}
|
||||
|
||||
await page.waitForNetworkIdle({ idleTime: 2000, timeout: 0 });
|
||||
|
||||
let jobLinks: string[] = [];
|
||||
|
||||
while (jobLinks.length < 1) {
|
||||
await randomDelay(1000, 2000);
|
||||
jobLinks = await browserService.getJobLinks();
|
||||
console.log(`Found ${jobLinks.length} job links`);
|
||||
}
|
||||
|
||||
for (const link of jobLinks) {
|
||||
try {
|
||||
const job = await browserService.getJobData(link);
|
||||
|
||||
const jobId = await saveJob(job);
|
||||
|
||||
if (jobId) {
|
||||
await saveJobDescription({
|
||||
criteria: job.criteria,
|
||||
description: job.description || "",
|
||||
jobId: jobId,
|
||||
});
|
||||
}
|
||||
await randomDelay(1000, 2000);
|
||||
} catch (error) {
|
||||
console.error(`Error processing job link ${link}:`, error);
|
||||
await randomDelay(1000, 2000);
|
||||
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
console.log("Saved job listings to the database");
|
||||
|
||||
console.log("Saved job descriptions to the database");
|
||||
|
||||
await randomDelay(1500, 2500);
|
||||
} catch (error) {
|
||||
console.error("Error scraping hiring cafe jobs:", error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (globalBrowser) {
|
||||
console.log("Closing browser on app shutdown");
|
||||
await globalBrowser.close();
|
||||
globalBrowser = null;
|
||||
}
|
||||
|
||||
if (chrome) {
|
||||
chrome.kill("SIGINT");
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log("Starting Hiring Cafe scraper service...");
|
||||
await scrapeHiringCafeJobs();
|
||||
}
|
||||
|
||||
process.on("SIGINT", async () => {
|
||||
console.log("Received SIGINT signal");
|
||||
await cleanup();
|
||||
if (chrome) {
|
||||
chrome.kill("SIGINT");
|
||||
}
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
process.on("SIGTERM", async () => {
|
||||
console.log("Received SIGTERM signal");
|
||||
await cleanup();
|
||||
if (chrome) {
|
||||
chrome.kill("SIGTERM");
|
||||
}
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
main().catch((error) => {
|
||||
console.error("Fatal error:", error);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/infrastructure"
|
||||
"github.com/jobs-scraper/libs/repo"
|
||||
"github.com/jobs-scraper/services/scraper-hiringcafe/models"
|
||||
"github.com/jobs-scraper/services/scraper-hiringcafe/scraper"
|
||||
"github.com/joho/godotenv"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := godotenv.Load(".local.env"); err != nil {
|
||||
log.Println("No .local.env file found, trying .env")
|
||||
if err := godotenv.Load(); err != nil {
|
||||
log.Println("No .env file found, using system environment variables")
|
||||
}
|
||||
}
|
||||
|
||||
// Setup database connection
|
||||
dbConfig := infrastructure.LoadConfigFromEnv()
|
||||
db, err := infrastructure.NewConnection(dbConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Error connecting to db: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
if err := db.Ping(); err != nil {
|
||||
log.Fatalf("Error pinging db: %v", err)
|
||||
}
|
||||
log.Println("Successfully connected to db")
|
||||
|
||||
// Initialize repositories
|
||||
jobRepo := repo.NewJobRepository(db)
|
||||
jobDescRepo := repo.NewJobDescriptionRepository(db)
|
||||
|
||||
// Example filters - these will be configurable later
|
||||
filters := models.SearchFilters{
|
||||
Keywords: "software engineer",
|
||||
Location: "",
|
||||
Remote: true,
|
||||
}
|
||||
|
||||
// Create scraper and run
|
||||
s := scraper.New(jobRepo, jobDescRepo)
|
||||
|
||||
// Handle graceful shutdown
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
|
||||
|
||||
go func() {
|
||||
<-sigChan
|
||||
log.Println("Received shutdown signal, cleaning up...")
|
||||
cancel()
|
||||
}()
|
||||
|
||||
if err := s.Scrape(ctx, filters); err != nil {
|
||||
log.Printf("Error scraping: %v", err)
|
||||
}
|
||||
|
||||
log.Println("Scraper finished")
|
||||
}
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
package models
|
||||
|
||||
// SearchFilters holds the filter configuration for hiring.cafe
|
||||
type SearchFilters struct {
|
||||
Keywords string
|
||||
Location string
|
||||
Remote bool
|
||||
JobType string // full-time, part-time, contract, internship
|
||||
DatePosted string // 24h, 7d, 30d
|
||||
}
|
||||
|
||||
// ScrapedJob holds the raw scraped job data before conversion to domain.Job
|
||||
type ScrapedJob struct {
|
||||
Title string
|
||||
Company string
|
||||
CompanyLink string
|
||||
Location string
|
||||
JobLink string
|
||||
PostedTime string
|
||||
Description string
|
||||
Criteria map[string]string
|
||||
}
|
||||
1104
services/scraper-hiringcafe/package-lock.json
generated
Normal file
1104
services/scraper-hiringcafe/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load diff
27
services/scraper-hiringcafe/package.json
Normal file
27
services/scraper-hiringcafe/package.json
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
{
|
||||
"name": "glassdoor-scraper",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "dist/index.js",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"start": "node dist/index.js",
|
||||
"dev": "ts-node index.ts",
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@types/amqplib": "^0.10.7",
|
||||
"amqplib": "^0.10.9",
|
||||
"dotenv": "^17.2.3",
|
||||
"postgres": "^3.4.7",
|
||||
"puppeteer-core": "^24.25.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.8.1",
|
||||
"ts-node": "^10.9.2",
|
||||
"typescript": "^5.9.3"
|
||||
}
|
||||
}
|
||||
33
services/scraper-hiringcafe/queries/save-job-description.ts
Normal file
33
services/scraper-hiringcafe/queries/save-job-description.ts
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
import sql from "../db/db";
|
||||
import { Job } from "../types/job";
|
||||
import { JobDescription } from "../types/job-description";
|
||||
|
||||
export const saveJobDescription = async (
|
||||
job: JobDescription
|
||||
): Promise<void> => {
|
||||
if (!job.jobId) {
|
||||
console.log("Job has no id, cannot save description.");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await sql`
|
||||
INSERT INTO public.job_descriptions (
|
||||
job_id,
|
||||
description,
|
||||
job_criteria
|
||||
) VALUES (
|
||||
${job.jobId},
|
||||
${job.description},
|
||||
${JSON.stringify(job.criteria)}
|
||||
)
|
||||
ON CONFLICT (job_id) DO UPDATE SET
|
||||
description = EXCLUDED.description,
|
||||
job_criteria = EXCLUDED.job_criteria,
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
`;
|
||||
console.log(`Updated description for job ${job.jobId}.`);
|
||||
} catch (error) {
|
||||
console.error("Error updating job description:", error);
|
||||
}
|
||||
};
|
||||
42
services/scraper-hiringcafe/queries/save-jobs.ts
Normal file
42
services/scraper-hiringcafe/queries/save-jobs.ts
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
import sql from "../db/db";
|
||||
import { Job } from "../types/job";
|
||||
|
||||
export const saveJob = async (job: Job): Promise<number | null> => {
|
||||
try {
|
||||
const result = await sql`
|
||||
INSERT INTO public.jobs (
|
||||
title,
|
||||
company,
|
||||
location,
|
||||
company_link,
|
||||
job_link,
|
||||
provider,
|
||||
status,
|
||||
job_timestamp
|
||||
) VALUES (
|
||||
${job.title},
|
||||
${job.company},
|
||||
${job.location},
|
||||
${job.companyLink},
|
||||
${job.jobLink},
|
||||
${job.provider},
|
||||
${job.status},
|
||||
${job.jobPostTime}
|
||||
)
|
||||
ON CONFLICT (job_link) DO UPDATE SET
|
||||
title = EXCLUDED.title,
|
||||
company = EXCLUDED.company,
|
||||
company_link = EXCLUDED.company_link,
|
||||
location = EXCLUDED.location,
|
||||
job_timestamp = EXCLUDED.job_timestamp
|
||||
RETURNING id
|
||||
`;
|
||||
|
||||
const id = result[0]?.id ?? null;
|
||||
console.log(`Saved job with id ${id} to the database.`);
|
||||
return id;
|
||||
} catch (error) {
|
||||
console.error("Error saving job to the database:", error);
|
||||
return null;
|
||||
}
|
||||
};
|
||||
Binary file not shown.
|
|
@ -1,570 +0,0 @@
|
|||
package scraper
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/libs/repo"
|
||||
"github.com/jobs-scraper/services/scraper-hiringcafe/api"
|
||||
"github.com/jobs-scraper/services/scraper-hiringcafe/browser"
|
||||
"github.com/jobs-scraper/services/scraper-hiringcafe/models"
|
||||
)
|
||||
|
||||
// HiringCafeScraper handles scraping jobs from hiring.cafe
|
||||
type HiringCafeScraper struct {
|
||||
jobRepo *repo.JobRepository
|
||||
jobDescRepo *repo.JobDescriptionRepository
|
||||
apiClient *api.Client
|
||||
browser *browser.Browser
|
||||
}
|
||||
|
||||
// New creates a new scraper instance
|
||||
func New(jobRepo *repo.JobRepository, jobDescRepo *repo.JobDescriptionRepository) *HiringCafeScraper {
|
||||
return &HiringCafeScraper{
|
||||
jobRepo: jobRepo,
|
||||
jobDescRepo: jobDescRepo,
|
||||
apiClient: api.NewClient(),
|
||||
}
|
||||
}
|
||||
|
||||
// Scrape performs the main scraping operation using the API
|
||||
func (s *HiringCafeScraper) Scrape(ctx context.Context, filters models.SearchFilters) error {
|
||||
log.Println("Starting hiring.cafe scraper using API...")
|
||||
|
||||
// Convert models.SearchFilters to api.SearchFilters
|
||||
apiFilters := api.SearchFilters{
|
||||
Query: filters.Keywords,
|
||||
Country: "United States",
|
||||
Remote: filters.Remote,
|
||||
Hybrid: true,
|
||||
Onsite: true,
|
||||
FullTime: true,
|
||||
PartTime: false,
|
||||
Contract: false,
|
||||
Internship: false,
|
||||
EntryLevel: true,
|
||||
MidLevel: true,
|
||||
SeniorLevel: false,
|
||||
DateRangeDays: 30,
|
||||
MaxPages: 10,
|
||||
}
|
||||
|
||||
if filters.Location != "" {
|
||||
apiFilters.Country = filters.Location
|
||||
}
|
||||
|
||||
log.Printf("Searching for: %s in %s", apiFilters.Query, apiFilters.Country)
|
||||
|
||||
// Fetch jobs from API
|
||||
apiJobs, err := s.apiClient.Search(apiFilters)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to fetch jobs from API: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("Found %d jobs from API", len(apiJobs))
|
||||
|
||||
if len(apiJobs) == 0 {
|
||||
log.Println("No jobs found")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Convert API jobs to scraped jobs
|
||||
var scrapedJobs []models.ScrapedJob
|
||||
for _, job := range apiJobs {
|
||||
scrapedJob := models.ScrapedJob{
|
||||
Title: job.Title,
|
||||
Company: job.Company,
|
||||
CompanyLink: job.CompanyURL,
|
||||
Location: job.Location,
|
||||
JobLink: job.ApplyURL,
|
||||
PostedTime: job.PostedAt.Format("2006-01-02"),
|
||||
Description: job.DescriptionClean,
|
||||
Criteria: make(map[string]string),
|
||||
}
|
||||
|
||||
if job.Salary != "" {
|
||||
scrapedJob.Criteria["Salary"] = job.Salary
|
||||
}
|
||||
if job.Remote {
|
||||
scrapedJob.Criteria["Remote"] = "Yes"
|
||||
}
|
||||
|
||||
scrapedJobs = append(scrapedJobs, scrapedJob)
|
||||
}
|
||||
|
||||
// Save jobs to database
|
||||
if err := s.saveJobs(scrapedJobs); err != nil {
|
||||
return fmt.Errorf("failed to save jobs: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("Successfully saved %d jobs to database", len(scrapedJobs))
|
||||
return nil
|
||||
}
|
||||
|
||||
// ScrapeWithBrowser performs scraping using browser automation (fallback if API fails)
|
||||
func (s *HiringCafeScraper) ScrapeWithBrowser(ctx context.Context, filters models.SearchFilters) error {
|
||||
// Create browser with anti-detection
|
||||
config := browser.DefaultConfig()
|
||||
config.Headless = false
|
||||
|
||||
b, err := browser.New(ctx, config)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create browser: %w", err)
|
||||
}
|
||||
defer b.Close()
|
||||
s.browser = b
|
||||
|
||||
log.Println("Starting browser and navigating to hiring.cafe...")
|
||||
|
||||
// Navigate to hiring.cafe
|
||||
if err := b.Navigate("https://hiring.cafe/"); err != nil {
|
||||
if err == browser.ErrCaptchaDetected {
|
||||
log.Println("CAPTCHA/Bot detection encountered - terminating scraper")
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("failed to navigate to hiring.cafe: %w", err)
|
||||
}
|
||||
|
||||
log.Println("Page loaded successfully, no CAPTCHA detected")
|
||||
|
||||
// Apply filters
|
||||
if err := s.applyFilters(filters); err != nil {
|
||||
return fmt.Errorf("failed to apply filters: %w", err)
|
||||
}
|
||||
|
||||
log.Println("Filters applied, extracting job listings...")
|
||||
|
||||
// Extract job listings
|
||||
jobs, err := s.extractJobListings()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to extract job listings: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("Found %d jobs", len(jobs))
|
||||
|
||||
// Save jobs to database
|
||||
if err := s.saveJobs(jobs); err != nil {
|
||||
return fmt.Errorf("failed to save jobs: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("Successfully saved %d jobs to database", len(jobs))
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyFilters applies search filters on hiring.cafe
|
||||
func (s *HiringCafeScraper) applyFilters(filters models.SearchFilters) error {
|
||||
if err := s.browser.Sleep(2 * time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Apply keyword search if provided
|
||||
if filters.Keywords != "" {
|
||||
log.Printf("Searching for: %s", filters.Keywords)
|
||||
|
||||
// Try multiple possible search input selectors
|
||||
searchSelectors := []string{
|
||||
`input[type="search"]`,
|
||||
`input[placeholder*="search" i]`,
|
||||
`input[placeholder*="job" i]`,
|
||||
`input[type="text"]`,
|
||||
`input[name="q"]`,
|
||||
`input[name="search"]`,
|
||||
}
|
||||
|
||||
var searchApplied bool
|
||||
for _, selector := range searchSelectors {
|
||||
if err := s.browser.Type(selector, filters.Keywords); err == nil {
|
||||
searchApplied = true
|
||||
log.Printf("Applied search using selector: %s", selector)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if searchApplied {
|
||||
browser.RandomDelay(500, 1000)
|
||||
// Press Enter to search
|
||||
var result interface{}
|
||||
s.browser.Evaluate(`
|
||||
const input = document.querySelector('input[type="search"], input[placeholder*="search" i], input[type="text"]');
|
||||
if (input) {
|
||||
input.dispatchEvent(new KeyboardEvent('keydown', {key: 'Enter', code: 'Enter', keyCode: 13, which: 13, bubbles: true}));
|
||||
input.form?.submit();
|
||||
}
|
||||
`, &result)
|
||||
s.browser.Sleep(3 * time.Second)
|
||||
} else {
|
||||
log.Println("Warning: could not find search input")
|
||||
}
|
||||
}
|
||||
|
||||
// Apply remote filter if enabled
|
||||
if filters.Remote {
|
||||
log.Println("Applying remote filter...")
|
||||
remoteSelectors := []string{
|
||||
`[data-filter="remote"]`,
|
||||
`button:contains("Remote")`,
|
||||
`label:contains("Remote")`,
|
||||
`input[value="remote"]`,
|
||||
`[class*="remote"]`,
|
||||
`a[href*="remote"]`,
|
||||
}
|
||||
|
||||
for _, selector := range remoteSelectors {
|
||||
if err := s.browser.WaitAndClick(selector); err == nil {
|
||||
log.Printf("Applied remote filter using selector: %s", selector)
|
||||
s.browser.Sleep(2 * time.Second)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for results to load
|
||||
return s.browser.Sleep(3 * time.Second)
|
||||
}
|
||||
|
||||
// extractJobListings extracts job listings from the current page
|
||||
func (s *HiringCafeScraper) extractJobListings() ([]models.ScrapedJob, error) {
|
||||
if err := s.browser.Sleep(2 * time.Second); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Simulate scrolling to load more jobs
|
||||
s.browser.Evaluate(`window.scrollTo(0, document.body.scrollHeight / 2)`, nil)
|
||||
s.browser.Sleep(1 * time.Second)
|
||||
s.browser.Evaluate(`window.scrollTo(0, document.body.scrollHeight)`, nil)
|
||||
s.browser.Sleep(2 * time.Second)
|
||||
|
||||
// Extract job data using JavaScript
|
||||
var jobData []map[string]string
|
||||
err := s.browser.Evaluate(`
|
||||
(function() {
|
||||
const jobs = [];
|
||||
|
||||
// Try multiple selectors for job cards
|
||||
const selectors = [
|
||||
'[class*="job-card"]',
|
||||
'[class*="job-listing"]',
|
||||
'[class*="JobCard"]',
|
||||
'[class*="job-item"]',
|
||||
'article[class*="job"]',
|
||||
'div[data-job-id]',
|
||||
'a[href*="/jobs/"]',
|
||||
'a[href*="/job/"]',
|
||||
'[class*="listing-card"]',
|
||||
'[class*="position-card"]',
|
||||
'li[class*="job"]',
|
||||
'tr[class*="job"]'
|
||||
];
|
||||
|
||||
let elements = [];
|
||||
for (const selector of selectors) {
|
||||
const found = document.querySelectorAll(selector);
|
||||
if (found.length > 0) {
|
||||
elements = Array.from(found);
|
||||
console.log('Found jobs with selector:', selector, found.length);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// If no specific job cards found, try to find any links that look like job postings
|
||||
if (elements.length === 0) {
|
||||
const allLinks = document.querySelectorAll('a[href*="job"], a[href*="position"], a[href*="career"]');
|
||||
elements = Array.from(allLinks).filter(el => {
|
||||
const text = el.textContent.toLowerCase();
|
||||
return text.length > 10 && text.length < 200;
|
||||
});
|
||||
}
|
||||
|
||||
elements.forEach((el, index) => {
|
||||
try {
|
||||
// Try to extract job information
|
||||
const titleSelectors = ['h1', 'h2', 'h3', 'h4', '[class*="title"]', '[class*="Title"]', '[class*="name"]'];
|
||||
const companySelectors = ['[class*="company"]', '[class*="Company"]', '[class*="employer"]', '[class*="org"]'];
|
||||
const locationSelectors = ['[class*="location"]', '[class*="Location"]', '[class*="place"]', '[class*="city"]'];
|
||||
const timeSelectors = ['[class*="time"]', '[class*="date"]', '[class*="posted"]', 'time', '[class*="ago"]'];
|
||||
|
||||
let title = '';
|
||||
let company = '';
|
||||
let location = '';
|
||||
let postedTime = '';
|
||||
|
||||
for (const sel of titleSelectors) {
|
||||
const titleEl = el.querySelector(sel);
|
||||
if (titleEl && titleEl.textContent.trim()) {
|
||||
title = titleEl.textContent.trim();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
for (const sel of companySelectors) {
|
||||
const companyEl = el.querySelector(sel);
|
||||
if (companyEl && companyEl.textContent.trim()) {
|
||||
company = companyEl.textContent.trim();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
for (const sel of locationSelectors) {
|
||||
const locationEl = el.querySelector(sel);
|
||||
if (locationEl && locationEl.textContent.trim()) {
|
||||
location = locationEl.textContent.trim();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
for (const sel of timeSelectors) {
|
||||
const timeEl = el.querySelector(sel);
|
||||
if (timeEl && timeEl.textContent.trim()) {
|
||||
postedTime = timeEl.textContent.trim();
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Get job link
|
||||
let jobLink = '';
|
||||
const linkEl = el.tagName === 'A' ? el : el.querySelector('a[href*="job"], a[href*="position"], a');
|
||||
if (linkEl && linkEl.href) {
|
||||
jobLink = linkEl.href;
|
||||
}
|
||||
|
||||
// Get company link
|
||||
let companyLink = '';
|
||||
const companyLinkEl = el.querySelector('a[href*="company"], a[href*="employer"]');
|
||||
if (companyLinkEl && companyLinkEl.href) {
|
||||
companyLink = companyLinkEl.href;
|
||||
}
|
||||
|
||||
// If no title found, use link text
|
||||
if (!title && linkEl) {
|
||||
title = linkEl.textContent.trim();
|
||||
}
|
||||
|
||||
const job = {
|
||||
title: title.substring(0, 500),
|
||||
company: company.substring(0, 200),
|
||||
location: location.substring(0, 200),
|
||||
jobLink: jobLink,
|
||||
postedTime: postedTime.substring(0, 100),
|
||||
companyLink: companyLink
|
||||
};
|
||||
|
||||
// Only add if we have at least a title and link
|
||||
if (job.title && job.jobLink) {
|
||||
jobs.push(job);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error extracting job ' + index + ':', error);
|
||||
}
|
||||
});
|
||||
|
||||
// Remove duplicates based on jobLink
|
||||
const seen = new Set();
|
||||
return jobs.filter(job => {
|
||||
if (seen.has(job.jobLink)) return false;
|
||||
seen.add(job.jobLink);
|
||||
return true;
|
||||
});
|
||||
})()
|
||||
`, &jobData)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to extract job data: %w", err)
|
||||
}
|
||||
|
||||
// Convert to ScrapedJob structs
|
||||
var jobs []models.ScrapedJob
|
||||
for _, data := range jobData {
|
||||
job := models.ScrapedJob{
|
||||
Title: data["title"],
|
||||
Company: data["company"],
|
||||
CompanyLink: data["companyLink"],
|
||||
Location: data["location"],
|
||||
JobLink: data["jobLink"],
|
||||
PostedTime: data["postedTime"],
|
||||
Criteria: make(map[string]string),
|
||||
}
|
||||
jobs = append(jobs, job)
|
||||
}
|
||||
|
||||
return jobs, nil
|
||||
}
|
||||
|
||||
// getJobDetails navigates to a job page and extracts detailed information
|
||||
func (s *HiringCafeScraper) getJobDetails(job models.ScrapedJob) (models.ScrapedJob, error) {
|
||||
if job.JobLink == "" {
|
||||
return job, nil
|
||||
}
|
||||
|
||||
// Navigate to job detail page
|
||||
if err := s.browser.Navigate(job.JobLink); err != nil {
|
||||
return job, fmt.Errorf("failed to navigate to job page: %w", err)
|
||||
}
|
||||
|
||||
s.browser.Sleep(2 * time.Second)
|
||||
s.browser.SimulateHumanBehavior()
|
||||
|
||||
// Extract job description
|
||||
var description string
|
||||
s.browser.Evaluate(`
|
||||
(function() {
|
||||
const descSelectors = [
|
||||
'[class*="description"]',
|
||||
'[class*="Description"]',
|
||||
'[class*="job-content"]',
|
||||
'[class*="job-details"]',
|
||||
'[class*="details"]',
|
||||
'[id*="description"]',
|
||||
'article',
|
||||
'main',
|
||||
'.content'
|
||||
];
|
||||
|
||||
for (const selector of descSelectors) {
|
||||
const el = document.querySelector(selector);
|
||||
if (el && el.textContent.trim().length > 100) {
|
||||
return el.textContent.trim().substring(0, 10000);
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: get body text
|
||||
return document.body.innerText.substring(0, 10000);
|
||||
})()
|
||||
`, &description)
|
||||
|
||||
// Extract criteria
|
||||
var criteriaData map[string]string
|
||||
s.browser.Evaluate(`
|
||||
(function() {
|
||||
const criteria = {};
|
||||
|
||||
// Look for common job criteria patterns
|
||||
const patterns = [
|
||||
{key: 'Experience', selectors: ['[class*="experience"]', '[class*="Experience"]']},
|
||||
{key: 'Education', selectors: ['[class*="education"]', '[class*="Education"]']},
|
||||
{key: 'Skills', selectors: ['[class*="skills"]', '[class*="Skills"]']},
|
||||
{key: 'Salary', selectors: ['[class*="salary"]', '[class*="Salary"]', '[class*="compensation"]']},
|
||||
{key: 'JobType', selectors: ['[class*="job-type"]', '[class*="employment"]', '[class*="type"]']},
|
||||
];
|
||||
|
||||
patterns.forEach(({key, selectors}) => {
|
||||
for (const selector of selectors) {
|
||||
const el = document.querySelector(selector);
|
||||
if (el && el.textContent.trim()) {
|
||||
criteria[key] = el.textContent.trim().substring(0, 500);
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
return criteria;
|
||||
})()
|
||||
`, &criteriaData)
|
||||
|
||||
job.Description = description
|
||||
if criteriaData != nil {
|
||||
job.Criteria = criteriaData
|
||||
}
|
||||
|
||||
return job, nil
|
||||
}
|
||||
|
||||
// saveJobs saves the scraped jobs to the database
|
||||
func (s *HiringCafeScraper) saveJobs(scrapedJobs []models.ScrapedJob) error {
|
||||
if len(scrapedJobs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var domainJobs []domain.Job
|
||||
var jobDescriptions []domain.JobDescription
|
||||
|
||||
for _, sj := range scrapedJobs {
|
||||
postTime := parsePostedTime(sj.PostedTime)
|
||||
|
||||
job := domain.Job{
|
||||
Title: sj.Title,
|
||||
Company: sj.Company,
|
||||
CompanyLink: sj.CompanyLink,
|
||||
Location: sj.Location,
|
||||
JobLink: sj.JobLink,
|
||||
Provider: domain.HiringCafe,
|
||||
JobPostTime: postTime,
|
||||
Status: domain.JobStatusCreated,
|
||||
}
|
||||
domainJobs = append(domainJobs, job)
|
||||
}
|
||||
|
||||
// Save jobs
|
||||
if err := s.jobRepo.SaveJobs(domainJobs); err != nil {
|
||||
return fmt.Errorf("failed to save jobs: %w", err)
|
||||
}
|
||||
|
||||
// Collect job descriptions
|
||||
for _, sj := range scrapedJobs {
|
||||
if sj.Description == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
jobDesc := domain.JobDescription{
|
||||
Description: sj.Description,
|
||||
Criteria: sj.Criteria,
|
||||
}
|
||||
jobDescriptions = append(jobDescriptions, jobDesc)
|
||||
}
|
||||
|
||||
log.Printf("Job descriptions collected: %d (saving requires job IDs)", len(jobDescriptions))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// parsePostedTime converts relative time strings to actual dates
|
||||
func parsePostedTime(timeStr string) *time.Time {
|
||||
now := time.Now()
|
||||
timeStr = strings.ToLower(strings.TrimSpace(timeStr))
|
||||
|
||||
if timeStr == "" {
|
||||
return &now
|
||||
}
|
||||
|
||||
// Parse common patterns
|
||||
if strings.Contains(timeStr, "hour") || strings.HasSuffix(timeStr, "h") {
|
||||
var hours int
|
||||
fmt.Sscanf(timeStr, "%d", &hours)
|
||||
if hours > 0 {
|
||||
t := now.Add(-time.Duration(hours) * time.Hour)
|
||||
return &t
|
||||
}
|
||||
}
|
||||
|
||||
if strings.Contains(timeStr, "day") || strings.HasSuffix(timeStr, "d") {
|
||||
var days int
|
||||
fmt.Sscanf(timeStr, "%d", &days)
|
||||
if days > 0 {
|
||||
t := now.AddDate(0, 0, -days)
|
||||
return &t
|
||||
}
|
||||
}
|
||||
|
||||
if strings.Contains(timeStr, "week") || strings.HasSuffix(timeStr, "w") {
|
||||
var weeks int
|
||||
fmt.Sscanf(timeStr, "%d", &weeks)
|
||||
if weeks > 0 {
|
||||
t := now.AddDate(0, 0, -weeks*7)
|
||||
return &t
|
||||
}
|
||||
}
|
||||
|
||||
if strings.Contains(timeStr, "month") || strings.HasSuffix(timeStr, "m") {
|
||||
var months int
|
||||
fmt.Sscanf(timeStr, "%d", &months)
|
||||
if months > 0 {
|
||||
t := now.AddDate(0, -months, 0)
|
||||
return &t
|
||||
}
|
||||
}
|
||||
|
||||
return &now
|
||||
}
|
||||
99
services/scraper-hiringcafe/services/browserService.ts
Normal file
99
services/scraper-hiringcafe/services/browserService.ts
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
import { ElementHandle, Page } from "puppeteer-core";
|
||||
import { randomDelay } from "../utils/browser";
|
||||
import { Job } from "../types/job";
|
||||
import { createDateFromRelativeTime } from "../utils/parse-time";
|
||||
|
||||
export class BrowserService {
|
||||
private page: Page;
|
||||
|
||||
constructor(page: Page) {
|
||||
this.page = page;
|
||||
}
|
||||
|
||||
async getJobLinks(): Promise<string[]> {
|
||||
const jobLinks = await this.page.evaluate(() => {
|
||||
const links = document.querySelectorAll(
|
||||
"a[href*='/viewjob/']"
|
||||
) as NodeListOf<HTMLAnchorElement>;
|
||||
return Array.from(links).map((link) => link.href);
|
||||
});
|
||||
|
||||
return jobLinks;
|
||||
}
|
||||
|
||||
async getJobData(jobLink: string): Promise<Job> {
|
||||
let jobContainer: ElementHandle<Element> | null = null;
|
||||
|
||||
await this.page.goto(jobLink, {
|
||||
waitUntil: "domcontentloaded",
|
||||
timeout: 60000,
|
||||
});
|
||||
|
||||
// await this.page.waitForNetworkIdle({ idleTime: 3000, timeout: 0 });
|
||||
|
||||
try {
|
||||
jobContainer = await this.page.waitForSelector(
|
||||
"::-p-xpath(/html/body/div[1]/div[3]/div/div/div/div[1])",
|
||||
{
|
||||
timeout: 10000,
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
console.log("No job listings found on the page");
|
||||
return {} as Job;
|
||||
}
|
||||
|
||||
if (jobContainer === null) {
|
||||
console.log("Job container not found");
|
||||
return {} as Job;
|
||||
}
|
||||
|
||||
const jobTitle = await jobContainer.waitForSelector("h2");
|
||||
const jobTitleText = await jobTitle?.evaluate((el) =>
|
||||
el.textContent?.trim()
|
||||
);
|
||||
const company = await jobContainer.waitForSelector(
|
||||
"span.text-xl.font-semibold.text-gray-700.flex-none"
|
||||
);
|
||||
const companyText = await company?.evaluate((el) => el.textContent?.trim());
|
||||
const location = await jobContainer.waitForSelector(
|
||||
"::-p-xpath(/html/body/div[1]/div[3]/div/div/div/div[1]/div/div[2]/div[3]/span)"
|
||||
);
|
||||
const locationText = await location?.evaluate((el) =>
|
||||
el.textContent?.trim()
|
||||
);
|
||||
const jobPostTime = await jobContainer.waitForSelector(
|
||||
"::-p-xpath(/html/body/div[1]/div[3]/div/div/div/div[1]/div/div[2]/div[1]/div/span)"
|
||||
);
|
||||
const jobPostTimeText = await jobPostTime?.evaluate((el) =>
|
||||
el.textContent?.trim()
|
||||
);
|
||||
|
||||
const time = jobPostTimeText?.split(" ")[1] ?? "";
|
||||
|
||||
const parsedDateTime = createDateFromRelativeTime(time);
|
||||
|
||||
const description = await this.page.waitForSelector("article");
|
||||
|
||||
const descriptionText = await description?.evaluate((el) =>
|
||||
el.textContent?.trim()
|
||||
);
|
||||
|
||||
const job: Job = {
|
||||
company: companyText || "",
|
||||
title: jobTitleText || "",
|
||||
location: locationText || "",
|
||||
description: descriptionText || "",
|
||||
criteria: {},
|
||||
jobLink,
|
||||
companyLink: "",
|
||||
provider: 7,
|
||||
jobPostTime: parsedDateTime.toISOString(),
|
||||
status: 0,
|
||||
};
|
||||
|
||||
console.log({ job });
|
||||
|
||||
return job;
|
||||
}
|
||||
}
|
||||
35
services/scraper-hiringcafe/tsconfig.json
Normal file
35
services/scraper-hiringcafe/tsconfig.json
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2020",
|
||||
"module": "commonjs",
|
||||
"lib": ["ES2020", "DOM"],
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"removeComments": true,
|
||||
"noImplicitAny": true,
|
||||
"noImplicitReturns": true,
|
||||
"noImplicitThis": true,
|
||||
"noUnusedLocals": false,
|
||||
"noUnusedParameters": false,
|
||||
"exactOptionalPropertyTypes": false,
|
||||
"noImplicitOverride": true,
|
||||
"noPropertyAccessFromIndexSignature": false,
|
||||
"noUncheckedIndexedAccess": false
|
||||
},
|
||||
"include": [
|
||||
"**/*.ts",
|
||||
"**/*.tsx"
|
||||
],
|
||||
"exclude": [
|
||||
"node_modules",
|
||||
"dist"
|
||||
]
|
||||
}
|
||||
5
services/scraper-hiringcafe/types/job-description.ts
Normal file
5
services/scraper-hiringcafe/types/job-description.ts
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
export interface JobDescription {
|
||||
jobId: number;
|
||||
description: string;
|
||||
criteria: Record<string, string>;
|
||||
}
|
||||
13
services/scraper-hiringcafe/types/job.ts
Normal file
13
services/scraper-hiringcafe/types/job.ts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
export interface Job {
|
||||
id?: number;
|
||||
title: string;
|
||||
company: string;
|
||||
location: string;
|
||||
description: string;
|
||||
criteria: Record<string, string>;
|
||||
jobLink: string;
|
||||
companyLink: string;
|
||||
provider: 7;
|
||||
jobPostTime: string;
|
||||
status: number;
|
||||
}
|
||||
139
services/scraper-hiringcafe/utils/browser.ts
Normal file
139
services/scraper-hiringcafe/utils/browser.ts
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
import fs from "fs";
|
||||
import path from "path";
|
||||
import os from "os";
|
||||
import { spawn } from "child_process";
|
||||
import net from "net";
|
||||
|
||||
export interface ChromeVersionResponse {
|
||||
webSocketDebuggerUrl: string;
|
||||
}
|
||||
|
||||
const userDataDir = path.join(os.tmpdir(), "chrome-debug-profile");
|
||||
|
||||
export const getLocalISOString = (date = new Date()) => {
|
||||
const offset = date.getTimezoneOffset();
|
||||
const offsetAbs = Math.abs(offset);
|
||||
const hours = Math.floor(offsetAbs / 60)
|
||||
.toString()
|
||||
.padStart(2, "0");
|
||||
const minutes = (offsetAbs % 60).toString().padStart(2, "0");
|
||||
const sign = offset > 0 ? "-" : "+";
|
||||
|
||||
return new Date(date.getTime() - offset * 60000)
|
||||
.toISOString()
|
||||
.replace("Z", `${sign}${hours}:${minutes}`);
|
||||
};
|
||||
|
||||
function getChromePath(): string {
|
||||
const platform = os.platform();
|
||||
|
||||
const paths: Record<NodeJS.Platform, string[]> = {
|
||||
darwin: [
|
||||
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
||||
"/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
|
||||
"/Applications/Chromium.app/Contents/MacOS/Chromium",
|
||||
],
|
||||
win32: [
|
||||
process.env.LOCALAPPDATA + "\\Google\\Chrome\\Application\\chrome.exe",
|
||||
process.env.PROGRAMFILES + "\\Google\\Chrome\\Application\\chrome.exe",
|
||||
process.env["PROGRAMFILES(X86)"] +
|
||||
"\\Google\\Chrome\\Application\\chrome.exe",
|
||||
],
|
||||
linux: [
|
||||
"/usr/bin/google-chrome",
|
||||
"/usr/bin/google-chrome-stable",
|
||||
"/usr/bin/chromium",
|
||||
"/usr/bin/chromium-browser",
|
||||
],
|
||||
aix: [],
|
||||
freebsd: [],
|
||||
openbsd: [],
|
||||
sunos: [],
|
||||
android: [],
|
||||
haiku: [],
|
||||
cygwin: [],
|
||||
netbsd: [],
|
||||
};
|
||||
|
||||
const platformPaths = paths[platform] || [];
|
||||
|
||||
for (const path of platformPaths) {
|
||||
try {
|
||||
if (fs.existsSync(path)) {
|
||||
return path;
|
||||
}
|
||||
} catch (err) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error("Could not find Chrome/Chromium installation");
|
||||
}
|
||||
|
||||
export function launchChromeWithDebugging() {
|
||||
try {
|
||||
const chromePath = getChromePath();
|
||||
const args = [
|
||||
"--remote-debugging-port=9222",
|
||||
`--user-data-dir=${userDataDir}`,
|
||||
"--remote-allow-origins=*",
|
||||
"--incognito",
|
||||
// "--headless",
|
||||
// `--proxy-server=46.3.202.240:2120`,
|
||||
// `--proxy-server=http://jN8AqiMh:Pd9FEs6N@212.193.184.228:64616`,
|
||||
];
|
||||
|
||||
console.log(`Launching Chrome at: ${chromePath}`);
|
||||
|
||||
const chromeProcess = spawn(chromePath, args, {
|
||||
detached: true,
|
||||
stdio: "ignore",
|
||||
});
|
||||
|
||||
chromeProcess.unref();
|
||||
|
||||
chromeProcess.on("error", (err) => {
|
||||
console.error("Failed to start Chrome:", err);
|
||||
});
|
||||
|
||||
chromeProcess.on("exit", (code) => {
|
||||
if (code !== 0) {
|
||||
console.error(`Chrome process exited with code ${code}`);
|
||||
}
|
||||
});
|
||||
|
||||
console.log("Chrome launched successfully with debugging port 9222");
|
||||
return chromeProcess;
|
||||
} catch (error) {
|
||||
console.error("Failed to launch Chrome:", error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
export async function waitForChromeReady(port = 9222, timeout = 30000) {
|
||||
const startTime = Date.now();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
function check() {
|
||||
const client = net.createConnection({ port }, () => {
|
||||
client.end();
|
||||
resolve("");
|
||||
});
|
||||
|
||||
client.on("error", () => {
|
||||
if (Date.now() - startTime > timeout) {
|
||||
reject(new Error("Timeout waiting for Chrome to start"));
|
||||
} else {
|
||||
setTimeout(check, 500);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
check();
|
||||
});
|
||||
}
|
||||
|
||||
export function randomDelay(min: number, max: number): Promise<void> {
|
||||
const delay = Math.floor(Math.random() * (max - min) + min);
|
||||
return new Promise((resolve) => setTimeout(resolve, delay));
|
||||
}
|
||||
28
services/scraper-hiringcafe/utils/parse-time.ts
Normal file
28
services/scraper-hiringcafe/utils/parse-time.ts
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
export function createDateFromRelativeTime(text: string) {
|
||||
const now = new Date();
|
||||
const cleanText = text
|
||||
.toLowerCase()
|
||||
.replace("posted", "")
|
||||
.replace("ago", "")
|
||||
.trim();
|
||||
|
||||
// Parse common patterns
|
||||
const patterns = [
|
||||
{ regex: /(\d+)\s*d/, multiplier: 24 * 60 * 60 * 1000 },
|
||||
{ regex: /(\d+)\s*h/, multiplier: 60 * 60 * 1000 },
|
||||
{ regex: /(\d+)\s*hr/, multiplier: 60 * 60 * 1000 },
|
||||
{ regex: /(\d+)\s*min/, multiplier: 60 * 1000 },
|
||||
{ regex: /(\d+)\s*m(?!in)/, multiplier: 60 * 1000 },
|
||||
{ regex: /(\d+)\s*s/, multiplier: 1000 },
|
||||
];
|
||||
|
||||
for (const pattern of patterns) {
|
||||
const match = cleanText.match(pattern.regex);
|
||||
if (match) {
|
||||
const value = parseInt(match[1]);
|
||||
return new Date(now.getTime() - value * pattern.multiplier);
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(`Could not parse time string: ${text}`);
|
||||
}
|
||||
165
services/scraper-hiringcafe/utils/rabbitmq.ts
Normal file
165
services/scraper-hiringcafe/utils/rabbitmq.ts
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
import * as amqp from "amqplib";
|
||||
|
||||
export interface RabbitMQConfig {
|
||||
url: string;
|
||||
queueName: string;
|
||||
exchangeName: string;
|
||||
exchangeType: "direct" | "topic" | "fanout" | "headers";
|
||||
durable: boolean;
|
||||
}
|
||||
|
||||
export interface SearchQuery {
|
||||
keywords: string;
|
||||
location: string;
|
||||
fwt?: string;
|
||||
numPages: number;
|
||||
}
|
||||
|
||||
export class RabbitMQClient {
|
||||
private connection: amqp.ChannelModel | null = null;
|
||||
private channel: amqp.Channel | null = null;
|
||||
private config: RabbitMQConfig;
|
||||
|
||||
constructor(config: RabbitMQConfig) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
async connect(): Promise<void> {
|
||||
try {
|
||||
console.log(`Connecting to RabbitMQ at: ${this.config.url}`);
|
||||
this.connection = await amqp.connect(this.config.url);
|
||||
console.log("Connected to RabbitMQ successfully");
|
||||
|
||||
if (!this.connection) {
|
||||
throw new Error("Failed to establish RabbitMQ connection");
|
||||
}
|
||||
|
||||
this.channel = await this.connection.createChannel();
|
||||
|
||||
if (!this.channel) {
|
||||
throw new Error("Failed to create RabbitMQ channel");
|
||||
}
|
||||
|
||||
// Ensure exchange and queue exist
|
||||
await this.channel.assertExchange(
|
||||
this.config.exchangeName,
|
||||
this.config.exchangeType,
|
||||
{ durable: this.config.durable }
|
||||
);
|
||||
|
||||
await this.channel.assertQueue(this.config.queueName, {
|
||||
durable: this.config.durable,
|
||||
arguments: {
|
||||
"x-dead-letter-exchange": "scraper_dlx",
|
||||
"x-dead-letter-routing-key": this.config.queueName,
|
||||
"x-message-ttl": 24 * 60 * 60 * 1000, // 24 hours in milliseconds
|
||||
"x-max-retries": 3,
|
||||
},
|
||||
});
|
||||
|
||||
await this.channel.bindQueue(
|
||||
this.config.queueName,
|
||||
this.config.exchangeName,
|
||||
this.config.queueName
|
||||
);
|
||||
|
||||
// Set prefetch to process one message at a time
|
||||
await this.channel.prefetch(1);
|
||||
|
||||
// Handle connection events
|
||||
this.connection.on("close", () => {
|
||||
console.log("RabbitMQ connection closed");
|
||||
this.connection = null;
|
||||
this.channel = null;
|
||||
});
|
||||
|
||||
this.connection.on("error", (error: Error) => {
|
||||
console.error("RabbitMQ connection error:", error);
|
||||
this.connection = null;
|
||||
this.channel = null;
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Failed to connect to RabbitMQ:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async subscribe(
|
||||
messageHandler: (message: SearchQuery) => Promise<void>
|
||||
): Promise<void> {
|
||||
if (!this.channel) {
|
||||
throw new Error(
|
||||
"RabbitMQ channel not initialized. Call connect() first."
|
||||
);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`Waiting for messages from ${this.config.queueName}. To exit press CTRL+C`
|
||||
);
|
||||
|
||||
await this.channel.consume(
|
||||
this.config.queueName,
|
||||
async (msg: amqp.ConsumeMessage | null) => {
|
||||
if (msg && this.channel) {
|
||||
try {
|
||||
const messageContent = msg.content.toString();
|
||||
console.log(`Received message: ${messageContent}`);
|
||||
|
||||
const searchQuery: SearchQuery = JSON.parse(messageContent);
|
||||
console.log(`Processing Glassdoor scrape request:`, searchQuery);
|
||||
|
||||
// Process the message using the provided handler
|
||||
await messageHandler(searchQuery);
|
||||
|
||||
// Acknowledge the message on success
|
||||
this.channel.ack(msg);
|
||||
console.log("Message processed successfully");
|
||||
} catch (error) {
|
||||
console.error("Error processing message:", error);
|
||||
// Reject the message and don't requeue it
|
||||
if (this.channel) {
|
||||
this.channel.nack(msg, false, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
try {
|
||||
if (this.channel) {
|
||||
await this.channel.close();
|
||||
this.channel = null;
|
||||
}
|
||||
if (this.connection) {
|
||||
await this.connection.close();
|
||||
this.connection = null;
|
||||
}
|
||||
console.log("RabbitMQ connection closed gracefully");
|
||||
} catch (error) {
|
||||
console.error("Error closing RabbitMQ connection:", error);
|
||||
}
|
||||
}
|
||||
|
||||
isConnected(): boolean {
|
||||
return this.connection !== null && this.channel !== null;
|
||||
}
|
||||
}
|
||||
|
||||
export function createRabbitMQConfig(): RabbitMQConfig {
|
||||
return {
|
||||
url: process.env.RABBITMQ_URL || "amqp://guest:guest@localhost:5672/",
|
||||
queueName: process.env.GLASSDOOR_QUEUE_NAME || "scraper.glassdoor",
|
||||
exchangeName: process.env.SCRAPER_EXCHANGE_NAME || "scraper_exchange",
|
||||
exchangeType: "topic",
|
||||
durable: true,
|
||||
};
|
||||
}
|
||||
|
||||
export async function createRabbitMQClient(): Promise<RabbitMQClient> {
|
||||
const config = createRabbitMQConfig();
|
||||
const client = new RabbitMQClient(config);
|
||||
await client.connect();
|
||||
return client;
|
||||
}
|
||||
Loading…
Reference in a new issue