From e54e5fb9eaa88754a46fd430c3e6256662bc8914 Mon Sep 17 00:00:00 2001 From: Elshimy Ziad Magdy Taha Date: Sat, 1 Nov 2025 10:59:58 +0500 Subject: [PATCH] added monorepo architecture --- .gitignore | 2 +- Makefile | 24 +++++++++-- cmd/migrate/main.go | 40 ------------------ go.work | 2 +- {api => packages/api}/app/app.go | 2 +- .../api}/commands/job/create-job.go | 0 {api => packages/api}/main.go | 2 +- .../glassdoor-scraper}/.env.example | 0 .../glassdoor-scraper}/index.ts | 0 .../glassdoor-scraper}/package-lock.json | 0 .../glassdoor-scraper}/package.json | 0 .../services/browserService.ts | 0 .../glassdoor-scraper}/tsconfig.json | 0 .../glassdoor-scraper}/utils/browser.ts | 0 .../glassdoor-scraper}/utils/rabbitmq.ts | 0 {services => packages}/linked-scraper/.env | 0 {services => packages}/linked-scraper/go.mod | 0 {services => packages}/linked-scraper/go.sum | 0 {services => packages}/linked-scraper/main.go | 0 .../linked-scraper/pipeline/job_pipeline.go | 0 .../pipeline/job_pipeline_workers.go | 0 .../linked-scraper/pipeline/scraper.go | 0 {services => packages}/linked-scraper/scraper | Bin 23 files changed, 24 insertions(+), 48 deletions(-) delete mode 100644 cmd/migrate/main.go rename {api => packages/api}/app/app.go (93%) rename {api => packages/api}/commands/job/create-job.go (100%) rename {api => packages/api}/main.go (98%) rename {glassdoor-scraper => packages/glassdoor-scraper}/.env.example (100%) rename {glassdoor-scraper => packages/glassdoor-scraper}/index.ts (100%) rename {glassdoor-scraper => packages/glassdoor-scraper}/package-lock.json (100%) rename {glassdoor-scraper => packages/glassdoor-scraper}/package.json (100%) rename {glassdoor-scraper => packages/glassdoor-scraper}/services/browserService.ts (100%) rename {glassdoor-scraper => packages/glassdoor-scraper}/tsconfig.json (100%) rename {glassdoor-scraper => packages/glassdoor-scraper}/utils/browser.ts (100%) rename {glassdoor-scraper => packages/glassdoor-scraper}/utils/rabbitmq.ts (100%) rename {services => packages}/linked-scraper/.env (100%) rename {services => packages}/linked-scraper/go.mod (100%) rename {services => packages}/linked-scraper/go.sum (100%) rename {services => packages}/linked-scraper/main.go (100%) rename {services => packages}/linked-scraper/pipeline/job_pipeline.go (100%) rename {services => packages}/linked-scraper/pipeline/job_pipeline_workers.go (100%) rename {services => packages}/linked-scraper/pipeline/scraper.go (100%) rename {services => packages}/linked-scraper/scraper (100%) diff --git a/.gitignore b/.gitignore index 26f8a17..12cede3 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,7 @@ *.dylib *.test *.out -go.work +# go.work # Dependency directories vendor/ diff --git a/Makefile b/Makefile index 57e933c..a121f16 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,9 @@ # Variables APP_NAME=jobs-scraper -API_DIR=./api -SCRAPER_DIR=./services/scraper +API_DIR=./packages/api +SCRAPER_DIR=./packages/linked-scraper +GLASSDOOR_DIR=./packages/glassdoor-scraper DOCS_DIR=./docs BIN_DIR=./bin MAIN_FILE=$(API_DIR)/main.go @@ -32,10 +33,25 @@ scraper: ## Run the scraper service @cd $(SCRAPER_DIR) && go run main.go .PHONY: build-scraper -build-scraper: ## Build the scraper service - @echo "Building scraper service..." +build-scraper: ## Build the linked scraper service + @echo "Building linked scraper service..." @cd $(SCRAPER_DIR) && go build -o scraper main.go +.PHONY: glassdoor-scraper +glassdoor-scraper: ## Run the glassdoor scraper service + @echo "Running glassdoor scraper service..." + @cd $(GLASSDOOR_DIR) && npm run dev + +.PHONY: build-glassdoor +build-glassdoor: ## Build the glassdoor scraper service + @echo "Building glassdoor scraper service..." + @cd $(GLASSDOOR_DIR) && npm run build + +.PHONY: build-api +build-api: ## Build the API service + @echo "Building API service..." + @cd $(API_DIR) && go build -o api main.go + .PHONY: cron cron: ## Run the job analysis cron service @echo "Starting Job Analysis Cron Service..." diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go deleted file mode 100644 index 775fe8d..0000000 --- a/cmd/migrate/main.go +++ /dev/null @@ -1,40 +0,0 @@ -package main - -import ( - "log" - - "github.com/jobs-scraper/shared/infrastructure" - "github.com/joho/godotenv" -) - -func main() { - // Try to load .local.env first, then fallback to .env - if err := godotenv.Load(".local.env"); err != nil { - log.Println("No .local.env file found, trying .env") - if err := godotenv.Load(".env"); err != nil { - log.Println("No .env file found, using system environment variables") - } - } - - dbConfig := infrastructure.LoadConfigFromEnv() - - db, err := infrastructure.NewConnection(dbConfig) - if err != nil { - log.Fatalf("Error connecting to database: %v", err) - } - defer db.Close() - - err = db.Ping() - if err != nil { - log.Fatalf("Error pinging database: %v", err) - } - - log.Println("Successfully connected to database") - - // Run migrations - if err := infrastructure.RunMigrations(db); err != nil { - log.Fatalf("Failed to run migrations: %v", err) - } - - log.Println("Migrations completed successfully") -} diff --git a/go.work b/go.work index b7e725a..8e311d5 100644 --- a/go.work +++ b/go.work @@ -2,7 +2,7 @@ go 1.24.0 use ( . - ./services/linked-scraper + ./packages/linked-scraper ./shared/domain ./shared/infrastructure ./shared/ports diff --git a/api/app/app.go b/packages/api/app/app.go similarity index 93% rename from api/app/app.go rename to packages/api/app/app.go index c19fdc1..19765fe 100644 --- a/api/app/app.go +++ b/packages/api/app/app.go @@ -4,7 +4,7 @@ import ( "database/sql" "github.com/gorilla/mux" - "github.com/jobs-scraper/api/commands/job" + "github.com/jobs-scraper/packages/api/commands/job" "github.com/jobs-scraper/shared/infrastructure/rabbitmq" "github.com/jobs-scraper/shared/ports" "github.com/jobs-scraper/shared/repo" diff --git a/api/commands/job/create-job.go b/packages/api/commands/job/create-job.go similarity index 100% rename from api/commands/job/create-job.go rename to packages/api/commands/job/create-job.go diff --git a/api/main.go b/packages/api/main.go similarity index 98% rename from api/main.go rename to packages/api/main.go index d255101..e5816ab 100644 --- a/api/main.go +++ b/packages/api/main.go @@ -10,7 +10,7 @@ import ( "time" "github.com/gorilla/mux" - "github.com/jobs-scraper/api/app" + "github.com/jobs-scraper/packages/api/app" "github.com/jobs-scraper/shared/infrastructure" httpHandler "github.com/jobs-scraper/shared/infrastructure/http" "github.com/jobs-scraper/shared/infrastructure/rabbitmq" diff --git a/glassdoor-scraper/.env.example b/packages/glassdoor-scraper/.env.example similarity index 100% rename from glassdoor-scraper/.env.example rename to packages/glassdoor-scraper/.env.example diff --git a/glassdoor-scraper/index.ts b/packages/glassdoor-scraper/index.ts similarity index 100% rename from glassdoor-scraper/index.ts rename to packages/glassdoor-scraper/index.ts diff --git a/glassdoor-scraper/package-lock.json b/packages/glassdoor-scraper/package-lock.json similarity index 100% rename from glassdoor-scraper/package-lock.json rename to packages/glassdoor-scraper/package-lock.json diff --git a/glassdoor-scraper/package.json b/packages/glassdoor-scraper/package.json similarity index 100% rename from glassdoor-scraper/package.json rename to packages/glassdoor-scraper/package.json diff --git a/glassdoor-scraper/services/browserService.ts b/packages/glassdoor-scraper/services/browserService.ts similarity index 100% rename from glassdoor-scraper/services/browserService.ts rename to packages/glassdoor-scraper/services/browserService.ts diff --git a/glassdoor-scraper/tsconfig.json b/packages/glassdoor-scraper/tsconfig.json similarity index 100% rename from glassdoor-scraper/tsconfig.json rename to packages/glassdoor-scraper/tsconfig.json diff --git a/glassdoor-scraper/utils/browser.ts b/packages/glassdoor-scraper/utils/browser.ts similarity index 100% rename from glassdoor-scraper/utils/browser.ts rename to packages/glassdoor-scraper/utils/browser.ts diff --git a/glassdoor-scraper/utils/rabbitmq.ts b/packages/glassdoor-scraper/utils/rabbitmq.ts similarity index 100% rename from glassdoor-scraper/utils/rabbitmq.ts rename to packages/glassdoor-scraper/utils/rabbitmq.ts diff --git a/services/linked-scraper/.env b/packages/linked-scraper/.env similarity index 100% rename from services/linked-scraper/.env rename to packages/linked-scraper/.env diff --git a/services/linked-scraper/go.mod b/packages/linked-scraper/go.mod similarity index 100% rename from services/linked-scraper/go.mod rename to packages/linked-scraper/go.mod diff --git a/services/linked-scraper/go.sum b/packages/linked-scraper/go.sum similarity index 100% rename from services/linked-scraper/go.sum rename to packages/linked-scraper/go.sum diff --git a/services/linked-scraper/main.go b/packages/linked-scraper/main.go similarity index 100% rename from services/linked-scraper/main.go rename to packages/linked-scraper/main.go diff --git a/services/linked-scraper/pipeline/job_pipeline.go b/packages/linked-scraper/pipeline/job_pipeline.go similarity index 100% rename from services/linked-scraper/pipeline/job_pipeline.go rename to packages/linked-scraper/pipeline/job_pipeline.go diff --git a/services/linked-scraper/pipeline/job_pipeline_workers.go b/packages/linked-scraper/pipeline/job_pipeline_workers.go similarity index 100% rename from services/linked-scraper/pipeline/job_pipeline_workers.go rename to packages/linked-scraper/pipeline/job_pipeline_workers.go diff --git a/services/linked-scraper/pipeline/scraper.go b/packages/linked-scraper/pipeline/scraper.go similarity index 100% rename from services/linked-scraper/pipeline/scraper.go rename to packages/linked-scraper/pipeline/scraper.go diff --git a/services/linked-scraper/scraper b/packages/linked-scraper/scraper similarity index 100% rename from services/linked-scraper/scraper rename to packages/linked-scraper/scraper