145 lines
3.5 KiB
Go
145 lines
3.5 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/jobs-scraper/internal/pkg/browser"
|
|
"github.com/jobs-scraper/internal/pkg/domain"
|
|
"github.com/jobs-scraper/libs/repo"
|
|
"github.com/jobs-scraper/services/scraper-google/analyzer"
|
|
si "github.com/jobs-scraper/services/scraper-google/setup-infrastructure"
|
|
"github.com/jobs-scraper/services/scraper-google/utils"
|
|
"github.com/levmv/sked"
|
|
)
|
|
|
|
type SearchResult struct {
|
|
Title string
|
|
Description string
|
|
Link string
|
|
CompanyName string
|
|
}
|
|
|
|
func main() {
|
|
db, err := si.SetupInfrastructure()
|
|
if err != nil {
|
|
log.Fatalf("Failed to set up infrastructure: %v", err)
|
|
}
|
|
|
|
jobRepo := repo.NewJobRepository(db)
|
|
jobDescRepo := repo.NewJobDescriptionRepository(db)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
// Launch Chrome once and share across all goroutines
|
|
browser, err := browser.NewBrowser(ctx)
|
|
|
|
if err != nil {
|
|
log.Fatalf("Failed to launch Chrome: %v", err)
|
|
}
|
|
|
|
defer browser.Close()
|
|
|
|
log.Printf("Chrome launched with WebSocket URL: %s", browser.Instance.WSURL)
|
|
|
|
sched := sked.New(ctx)
|
|
googleLinkStream := make(chan domain.GoogleLink, 100)
|
|
pages := []int{0, 10, 20, 30, 40}
|
|
// pages := []int{100}
|
|
|
|
log.Println("Starting scraper with scheduler...")
|
|
|
|
for _, page := range pages {
|
|
go func() {
|
|
log.Printf("Running initial scrape for page offset: %d, inital run", page)
|
|
Work(ctx, browser)(page, googleLinkStream)
|
|
}()
|
|
|
|
}
|
|
|
|
for _, page := range pages {
|
|
sched.Schedule(func(ctx context.Context) {
|
|
log.Printf("Running scheduled scrape for page offset: %d", page)
|
|
Work(ctx, browser)(page, googleLinkStream)
|
|
}).Every(time.Hour * 24)
|
|
}
|
|
|
|
go func() {
|
|
for googleLink := range googleLinkStream {
|
|
log.Printf("Received job: %s ", googleLink.CompanyName)
|
|
|
|
exists, err := jobRepo.JobExistsByLink(googleLink.Link)
|
|
|
|
if err != nil {
|
|
log.Printf("Error checking if job exists for link %s: %v", googleLink.Link, err)
|
|
continue
|
|
}
|
|
|
|
if exists {
|
|
log.Printf("Job already exists, skipping: %s", googleLink.Link)
|
|
continue
|
|
}
|
|
|
|
htmlRaw, url, err := utils.GetHTMLRaw(ctx, browser, googleLink)
|
|
|
|
if err != nil {
|
|
log.Printf("Error fetching HTML raw for link %s: %v", googleLink.Link, err)
|
|
continue
|
|
}
|
|
|
|
result, err := analyzer.GetJobDescription(htmlRaw)
|
|
if err != nil {
|
|
log.Printf("Error analyzing job description for link %s: %v", googleLink.Link, err)
|
|
continue
|
|
}
|
|
|
|
now := time.Now()
|
|
|
|
// Save the main job
|
|
job := domain.Job{
|
|
Title: googleLink.Title,
|
|
Company: googleLink.CompanyName,
|
|
CompanyLink: url,
|
|
Location: result.Criteria["location"],
|
|
JobLink: googleLink.Link,
|
|
Provider: domain.Google,
|
|
JobPostTime: &now,
|
|
Status: domain.JobStatusCreated,
|
|
}
|
|
|
|
jobID, err := jobRepo.SaveJob(job)
|
|
if err != nil {
|
|
log.Printf("Failed to save job: %v", err)
|
|
continue
|
|
}
|
|
|
|
// Save job description
|
|
jobDescription := domain.JobDescription{
|
|
JobID: jobID,
|
|
Description: result.Description,
|
|
Criteria: result.Criteria,
|
|
}
|
|
|
|
if err := jobDescRepo.SaveJobDescriptions([]domain.JobDescription{jobDescription}); err != nil {
|
|
log.Printf("Failed to save job description for job %d: %v", jobID, err)
|
|
continue
|
|
}
|
|
|
|
log.Printf("Successfully saved job: %s from %s (ID: %d)", job.Title, job.Company, jobID)
|
|
}
|
|
}()
|
|
|
|
if err := sched.Run(); err != nil {
|
|
log.Fatalf("Failed to start scheduler: %v", err)
|
|
}
|
|
|
|
c := make(chan os.Signal, 1)
|
|
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
|
|
<-c
|
|
cancel()
|
|
log.Println("Shutting down scraper...")
|
|
}
|