112 lines
2.8 KiB
Go
112 lines
2.8 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/jobs-scraper/internal/pkg/domain"
|
|
"github.com/jobs-scraper/libs/repo"
|
|
"github.com/jobs-scraper/services/scraper-google/analyzer"
|
|
si "github.com/jobs-scraper/services/scraper-google/setup-infrastructure"
|
|
"github.com/jobs-scraper/services/scraper-google/utils"
|
|
"github.com/levmv/sked"
|
|
)
|
|
|
|
type SearchResult struct {
|
|
Title string
|
|
Description string
|
|
Link string
|
|
CompanyName string
|
|
}
|
|
|
|
func main() {
|
|
db, err := si.SetupInfrastructure()
|
|
if err != nil {
|
|
log.Fatalf("Failed to set up infrastructure: %v", err)
|
|
}
|
|
|
|
jobRepo := repo.NewJobRepository(db)
|
|
jobDescRepo := repo.NewJobDescriptionRepository(db)
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
sched := sked.New(ctx)
|
|
googleLinkStream := make(chan domain.GoogleLink, 100)
|
|
// pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}
|
|
pages := []int{90}
|
|
|
|
log.Println("Starting scraper with scheduler...")
|
|
|
|
for _, page := range pages {
|
|
// Work(ctx)(page, googleLinkStream)
|
|
sched.Schedule(func(ctx context.Context) {
|
|
log.Printf("Running scrape for page offset: %d", page)
|
|
Work(ctx)(page, googleLinkStream)
|
|
}).Every(24 * time.Hour)
|
|
}
|
|
|
|
if err := sched.Run(); err != nil {
|
|
log.Fatalf("Failed to start scheduler: %v", err)
|
|
}
|
|
|
|
go func() {
|
|
for googleLink := range googleLinkStream {
|
|
htmlRaw, url, err := utils.GetHTMLRaw(ctx, googleLink)
|
|
|
|
if err != nil {
|
|
log.Printf("Error fetching HTML raw for link %s: %v", googleLink.Link, err)
|
|
continue
|
|
}
|
|
|
|
result, err := analyzer.GetJobDescription(htmlRaw)
|
|
|
|
if err != nil {
|
|
log.Printf("Error analyzing job description for link %s: %v", googleLink.Link, err)
|
|
continue
|
|
}
|
|
|
|
now := time.Now()
|
|
|
|
job := domain.Job{
|
|
ID: googleLink.ID,
|
|
Title: googleLink.Title,
|
|
Company: googleLink.CompanyName,
|
|
CompanyLink: googleLink.Link,
|
|
Location: result.Criteria["location"],
|
|
JobLink: url,
|
|
Provider: domain.Google,
|
|
JobPostTime: &now,
|
|
Status: domain.JobStatusCreated,
|
|
}
|
|
|
|
jobDescription := domain.JobDescription{
|
|
JobID: googleLink.ID,
|
|
Description: result.Description,
|
|
Criteria: result.Criteria,
|
|
}
|
|
|
|
if err := jobRepo.SaveJobs([]domain.Job{job}); err != nil {
|
|
log.Printf("Failed to save job %d: %v", googleLink.ID, err)
|
|
continue
|
|
}
|
|
|
|
if err := jobDescRepo.SaveJobDescriptions([]domain.JobDescription{jobDescription}); err != nil {
|
|
log.Printf("Failed to save job description for job %d: %v", googleLink.ID, err)
|
|
continue
|
|
}
|
|
|
|
log.Printf("Successfully saved job: %s from %s (ID: %d)", job.Title, job.Company, job.ID)
|
|
}
|
|
}()
|
|
|
|
c := make(chan os.Signal, 1)
|
|
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
|
|
<-c
|
|
cancel()
|
|
log.Println("Shutting down scraper...")
|
|
|
|
}
|