bugblitz77/jobs-monorepo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix schedule

Browse source
This commit is contained in:
Elshimy Ziad Magdy Taha 2025-12-15 13:02:28 +05:00
parent 4bbf4243c7
commit 6154e95d32
1 changed files with 12 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
services/scraper-google/main.go
View file

@ -36,21 +36,20 @@ func main() {
sched := sked.New(ctx)
googleLinkStream := make(chan domain.GoogleLink, 100)
// pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}
pages := []int{90}
pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}
log.Println("Starting scraper with scheduler...")
for _, page := range pages {
// Work(ctx)(page, googleLinkStream)
sched.Schedule(func(ctx context.Context) {
log.Printf("Running scrape for page offset: %d", page)
go func() {
log.Printf("Running initial scrape for page offset: %d", page)
Work(ctx)(page, googleLinkStream)
}).Every(24 * time.Hour)
}
}()
if err := sched.Run(); err != nil {
log.Fatalf("Failed to start scheduler: %v", err)
sched.Schedule(func(ctx context.Context) {
log.Printf("Running scheduled scrape for page offset: %d", page)
Work(ctx)(page, googleLinkStream)
}).Every(time.Minute * 1)
}
go func() {
@ -103,6 +102,10 @@ func main() {
}
}()
if err := sched.Run(); err != nil {
log.Fatalf("Failed to start scheduler: %v", err)
}
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
<-c