From 6154e95d32329ae69dd79b53da0bad7c56b61360 Mon Sep 17 00:00:00 2001
From: Elshimy Ziad Magdy Taha <z.elshimy@kari.com>
Date: Mon, 15 Dec 2025 13:02:28 +0500
Subject: [PATCH] fix schedule

---
 services/scraper-google/main.go | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/services/scraper-google/main.go b/services/scraper-google/main.go
index 8e1e530..62501fd 100644
--- a/services/scraper-google/main.go
+++ b/services/scraper-google/main.go
@@ -36,21 +36,20 @@ func main() {
 
 	sched := sked.New(ctx)
 	googleLinkStream := make(chan domain.GoogleLink, 100)
-	// pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}
-	pages := []int{90}
+	pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}
 
 	log.Println("Starting scraper with scheduler...")
 
 	for _, page := range pages {
-		// Work(ctx)(page, googleLinkStream)
-		sched.Schedule(func(ctx context.Context) {
-			log.Printf("Running scrape for page offset: %d", page)
+		go func() {
+			log.Printf("Running initial scrape for page offset: %d", page)
 			Work(ctx)(page, googleLinkStream)
-		}).Every(24 * time.Hour)
-	}
+		}()
 
-	if err := sched.Run(); err != nil {
-		log.Fatalf("Failed to start scheduler: %v", err)
+		sched.Schedule(func(ctx context.Context) {
+			log.Printf("Running scheduled scrape for page offset: %d", page)
+			Work(ctx)(page, googleLinkStream)
+		}).Every(time.Minute * 1)
 	}
 
 	go func() {
@@ -103,6 +102,10 @@ func main() {
 		}
 	}()
 
+	if err := sched.Run(); err != nil {
+		log.Fatalf("Failed to start scheduler: %v", err)
+	}
+
 	c := make(chan os.Signal, 1)
 	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
 	<-c