diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..df22772 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,31 @@ +# Git +.git +.gitignore + +# Documentation +README.md +*.md + +# IDE +.vscode +.idea + +# Logs +*.log + +# OS +.DS_Store +Thumbs.db + +# Development files +.local.env + +# Playwright +.playwright-mcp + +# Node modules (if any) +node_modules/ + +# Temporary files +*.tmp +*.temp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..650d543 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,116 @@ +name: CI/CD Pipeline + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main ] + +jobs: + test: + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:15 + env: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: jobs_scraper_test + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + rabbitmq: + image: rabbitmq:3-management + ports: + - 5672:5672 + - 15672:15672 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: '1.24' + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '18' + + - name: Cache Go modules + uses: actions/cache@v3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Install dependencies + run: | + go mod download + cd services/scraper-glassdoor && npm ci + + - name: Run tests - API Service + run: | + cd services/api + go test ./... + + - name: Run tests - LinkedIn Scraper + run: | + cd services/scraper-linkedin + go test ./... + + - name: Run tests - Cron Analyzer + run: | + cd apps/cron-analyzer + go test ./... + + - name: Run tests - CV Analyzer + run: | + cd apps/cv-analyzer + go test ./... + + - name: Build all services + run: | + make build-api + make build-scraper-linkedin + make build-scraper-glassdoor + make build-cron-analyzer + make build-cv-analyzer + + docker: + needs: test + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and push API service + uses: docker/build-push-action@v5 + with: + context: ./services/api + push: true + tags: ${{ secrets.DOCKER_USERNAME }}/jobs-scraper-api:latest + + - name: Build and push LinkedIn scraper + uses: docker/build-push-action@v5 + with: + context: ./services/scraper-linkedin + push: true + tags: ${{ secrets.DOCKER_USERNAME }}/jobs-scraper-linkedin:latest diff --git a/Makefile b/Makefile index a121f16..f134f2c 100644 --- a/Makefile +++ b/Makefile @@ -2,17 +2,17 @@ # Variables APP_NAME=jobs-scraper -API_DIR=./packages/api -SCRAPER_DIR=./packages/linked-scraper -GLASSDOOR_DIR=./packages/glassdoor-scraper -DOCS_DIR=./docs +API_DIR=./services/api +SCRAPER_LINKEDIN_DIR=./services/scraper-linkedin +SCRAPER_GLASSDOOR_DIR=./services/scraper-glassdoor +CRON_ANALYZER_DIR=./apps/cron-analyzer +CV_ANALYZER_DIR=./apps/cv-analyzer +DOCS_DIR=./internal/swagger BIN_DIR=./bin -MAIN_FILE=$(API_DIR)/main.go -SCRAPER_MAIN_FILE=$(SCRAPER_DIR)/main.go -CRON_DIR=./cmd/cron-analyzer -CRON_MAIN_FILE=$(CRON_DIR)/main.go -MIGRATE_DIR=./cmd/migrate -MIGRATE_MAIN_FILE=$(MIGRATE_DIR)/main.go +API_MAIN_FILE=$(API_DIR)/cmd/server/main.go +SCRAPER_LINKEDIN_MAIN_FILE=$(SCRAPER_LINKEDIN_DIR)/main.go +CRON_MAIN_FILE=$(CRON_ANALYZER_DIR)/cmd/cli/main.go +CV_MAIN_FILE=$(CV_ANALYZER_DIR)/cmd/cli/main.go # Default target .DEFAULT_GOAL := help @@ -22,40 +22,67 @@ rabbitmq-server: @echo "Starting RabbitMQ server..." docker run --rm -p 5672:5672 -p 15672:15672 rabbitmq:3-management -.PHONY: run -run: ## Run the API application - @echo "Running $(APP_NAME) API..." - @go run $(MAIN_FILE) - -.PHONY: scraper -scraper: ## Run the scraper service - @echo "Running scraper service..." - @cd $(SCRAPER_DIR) && go run main.go - -.PHONY: build-scraper -build-scraper: ## Build the linked scraper service - @echo "Building linked scraper service..." - @cd $(SCRAPER_DIR) && go build -o scraper main.go - -.PHONY: glassdoor-scraper -glassdoor-scraper: ## Run the glassdoor scraper service - @echo "Running glassdoor scraper service..." - @cd $(GLASSDOOR_DIR) && npm run dev - -.PHONY: build-glassdoor -build-glassdoor: ## Build the glassdoor scraper service - @echo "Building glassdoor scraper service..." - @cd $(GLASSDOOR_DIR) && npm run build +# Services +.PHONY: run-api +run-api: ## Run the API service + @echo "Running $(APP_NAME) API service..." + @cd $(API_DIR) && go run cmd/server/main.go .PHONY: build-api build-api: ## Build the API service @echo "Building API service..." - @cd $(API_DIR) && go build -o api main.go + @cd $(API_DIR) && go build -o bin/api cmd/server/main.go + +.PHONY: run-scraper-linkedin +run-scraper-linkedin: ## Run the LinkedIn scraper service + @echo "Running LinkedIn scraper service..." + @cd $(SCRAPER_LINKEDIN_DIR) && go run main.go + +.PHONY: build-scraper-linkedin +build-scraper-linkedin: ## Build the LinkedIn scraper service + @echo "Building LinkedIn scraper service..." + @cd $(SCRAPER_LINKEDIN_DIR) && go build -o bin/scraper main.go + +.PHONY: run-scraper-glassdoor +run-scraper-glassdoor: ## Run the Glassdoor scraper service + @echo "Running Glassdoor scraper service..." + @cd $(SCRAPER_GLASSDOOR_DIR) && npm run dev + +.PHONY: build-scraper-glassdoor +build-scraper-glassdoor: ## Build the Glassdoor scraper service + @echo "Building Glassdoor scraper service..." + @cd $(SCRAPER_GLASSDOOR_DIR) && npm run build + +# Apps +.PHONY: run-cron-analyzer +run-cron-analyzer: ## Run the job analysis cron service + @echo "Starting Job Analysis Cron Service..." + @cd $(CRON_ANALYZER_DIR) && go run cmd/cli/main.go + +.PHONY: build-cron-analyzer +build-cron-analyzer: ## Build the cron analyzer app + @echo "Building cron analyzer app..." + @cd $(CRON_ANALYZER_DIR) && go build -o bin/cron-analyzer cmd/cli/main.go + +.PHONY: run-cv-analyzer +run-cv-analyzer: ## Run the CV analyzer app + @echo "Running CV analyzer app..." + @cd $(CV_ANALYZER_DIR) && go run cmd/cli/main.go + +.PHONY: build-cv-analyzer +build-cv-analyzer: ## Build the CV analyzer app + @echo "Building CV analyzer app..." + @cd $(CV_ANALYZER_DIR) && go build -o bin/cv-analyzer cmd/cli/main.go + +# Legacy aliases for backward compatibility +.PHONY: run +run: run-api ## Alias for run-api + +.PHONY: scraper +scraper: run-scraper-linkedin ## Alias for run-scraper-linkedin .PHONY: cron -cron: ## Run the job analysis cron service - @echo "Starting Job Analysis Cron Service..." - @go run $(CRON_MAIN_FILE) +cron: run-cron-analyzer ## Alias for run-cron-analyzer .PHONY: migrate migrate: ## Run database migrations @@ -79,7 +106,7 @@ help: ## Show this help message .PHONY: swagger swagger: ## Generate Swagger documentation @echo "Generating Swagger documentation..." - @go run github.com/swaggo/swag/cmd/swag@latest init -g $(MAIN_FILE) -o $(DOCS_DIR) + @go run github.com/swaggo/swag/cmd/swag@latest init -g $(API_MAIN_FILE) -o $(DOCS_DIR) @echo "Swagger documentation generated in $(DOCS_DIR)/" # Docker operations (if needed in the future) diff --git a/README.md b/README.md new file mode 100644 index 0000000..bf97c40 --- /dev/null +++ b/README.md @@ -0,0 +1,190 @@ +# Jobs Scraper Monorepo + +A modern monorepo architecture for job scraping and analysis services. + +## ๐Ÿ—๏ธ Architecture + +``` +jobs-scraper/ +โ”œโ”€โ”€ go.mod # Root go.mod with module name +โ”œโ”€โ”€ Makefile # Common build/test tasks +โ”œโ”€โ”€ .github/workflows/ # CI/CD pipelines +โ”œโ”€โ”€ tools/ # Build tools and utilities +โ”œโ”€โ”€ internal/ # Private shared packages +โ”‚ โ”œโ”€โ”€ pkg/ +โ”‚ โ”‚ โ”œโ”€โ”€ domain/ # Core domain models +โ”‚ โ”‚ โ”œโ”€โ”€ infrastructure/ # Database, RabbitMQ, HTTP +โ”‚ โ”‚ โ””โ”€โ”€ utils/ # Common utilities +โ”‚ โ””โ”€โ”€ migrations/ # Database migrations +โ”œโ”€โ”€ libs/ # Shared libraries +โ”‚ โ”œโ”€โ”€ ports/ # Command interfaces +โ”‚ โ”œโ”€โ”€ repo/ # Data repositories +โ”‚ โ””โ”€โ”€ server/ # Server utilities +โ”œโ”€โ”€ services/ # Deployable services +โ”‚ โ”œโ”€โ”€ api/ # HTTP API service +โ”‚ โ”œโ”€โ”€ scraper-linkedin/ # LinkedIn scraper service +โ”‚ โ””โ”€โ”€ scraper-glassdoor/ # Glassdoor scraper service +โ”œโ”€โ”€ apps/ # Client applications +โ”‚ โ”œโ”€โ”€ cron-analyzer/ # Job analysis cron +โ”‚ โ””โ”€โ”€ cv-analyzer/ # CV analysis utility +โ””โ”€โ”€ scripts/ # Deployment and utility scripts +``` + +## ๐Ÿš€ Services + +### API Service (`services/api/`) +- **Purpose**: HTTP API server with CQRS pattern +- **Tech Stack**: Go, Gorilla Mux, PostgreSQL, RabbitMQ +- **Features**: Job management, Swagger documentation, CORS support + +### LinkedIn Scraper (`services/scraper-linkedin/`) +- **Purpose**: Scrapes job postings from LinkedIn +- **Tech Stack**: Go, Playwright +- **Features**: Automated job scraping, RabbitMQ integration + +### Glassdoor Scraper (`services/scraper-glassdoor/`) +- **Purpose**: Scrapes job postings from Glassdoor +- **Tech Stack**: TypeScript, Node.js +- **Features**: Web scraping, data processing + +## ๐Ÿ“ฑ Applications + +### Cron Analyzer (`apps/cron-analyzer/`) +- **Purpose**: Analyzes jobs with AI on a schedule +- **Tech Stack**: Go, OpenRouter API +- **Features**: Job analysis, status updates + +### CV Analyzer (`apps/cv-analyzer/`) +- **Purpose**: Analyzes CVs against job requirements +- **Tech Stack**: Go, OpenRouter API +- **Features**: CV matching, skill analysis + +## ๐Ÿ› ๏ธ Development + +### Prerequisites +- Go 1.24+ +- Node.js 18+ +- Docker & Docker Compose +- PostgreSQL 15+ +- RabbitMQ 3+ + +### Quick Start + +1. **Clone the repository** + ```bash + git clone + cd jobs-scraper + ``` + +2. **Start infrastructure** + ```bash + make rabbitmq-server + # In another terminal: + docker run -d -p 5432:5432 -e POSTGRES_PASSWORD=password postgres:15 + ``` + +3. **Run services** + ```bash + # API Service + make run-api + + # LinkedIn Scraper + make run-scraper-linkedin + + # Glassdoor Scraper + make run-scraper-glassdoor + + # Cron Analyzer + make run-cron-analyzer + ``` + +### Available Commands + +#### Services +- `make run-api` - Run API service +- `make build-api` - Build API service +- `make run-scraper-linkedin` - Run LinkedIn scraper +- `make build-scraper-linkedin` - Build LinkedIn scraper +- `make run-scraper-glassdoor` - Run Glassdoor scraper +- `make build-scraper-glassdoor` - Build Glassdoor scraper + +#### Applications +- `make run-cron-analyzer` - Run cron analyzer +- `make build-cron-analyzer` - Build cron analyzer +- `make run-cv-analyzer` - Run CV analyzer +- `make build-cv-analyzer` - Build CV analyzer + +#### Infrastructure +- `make rabbitmq-server` - Start RabbitMQ server +- `make swagger` - Generate API documentation +- `make clean` - Clean build artifacts + +#### Legacy Aliases (Backward Compatibility) +- `make run` - Alias for `run-api` +- `make scraper` - Alias for `run-scraper-linkedin` +- `make cron` - Alias for `run-cron-analyzer` + +## ๐Ÿณ Docker + +Each service has its own Dockerfile and can be built independently: + +```bash +# Build API service +cd services/api +docker build -t jobs-scraper-api . + +# Build LinkedIn scraper +cd services/scraper-linkedin +docker build -t jobs-scraper-linkedin . +``` + +## ๐Ÿ”ง Configuration + +Environment variables are managed through `.env` files: + +- `.env` - Production configuration +- `.local.env` - Local development configuration + +## ๐Ÿงช Testing + +Run tests for individual services: + +```bash +# API Service +cd services/api && go test ./... + +# LinkedIn Scraper +cd services/scraper-linkedin && go test ./... + +# Cron Analyzer +cd apps/cron-analyzer && go test ./... +``` + +## ๐Ÿ“Š Monitoring + +- **API Documentation**: http://localhost:8080/swagger/ +- **RabbitMQ Management**: http://localhost:15672 (guest/guest) +- **Health Checks**: Built into each service + +## ๐Ÿš€ Deployment + +The monorepo includes GitHub Actions workflows for: +- Automated testing +- Docker image building +- Multi-service deployment + +## ๐Ÿค Contributing + +1. Fork the repository +2. Create a feature branch +3. Make your changes +4. Add tests +5. Submit a pull request + +## ๐Ÿ“„ License + +[Add your license information here] + +## ๐Ÿ†˜ Support + +For support and questions, please [create an issue](issues) or contact the development team. diff --git a/apps/cron-analyzer/bin/cron-analyzer b/apps/cron-analyzer/bin/cron-analyzer new file mode 100755 index 0000000..a6d70bc Binary files /dev/null and b/apps/cron-analyzer/bin/cron-analyzer differ diff --git a/cmd/cron-analyzer/main.go b/apps/cron-analyzer/cmd/cli/main.go similarity index 84% rename from cmd/cron-analyzer/main.go rename to apps/cron-analyzer/cmd/cli/main.go index b50f1cb..96840db 100644 --- a/cmd/cron-analyzer/main.go +++ b/apps/cron-analyzer/cmd/cli/main.go @@ -3,8 +3,8 @@ package main import ( "log" - "github.com/jobs-scraper/cron" - "github.com/jobs-scraper/shared/infrastructure" + "github.com/jobs-scraper/apps/cron-analyzer/internal" + "github.com/jobs-scraper/internal/pkg/infrastructure" "github.com/joho/godotenv" ) @@ -31,7 +31,7 @@ func main() { log.Println("Successfully connected to db") - analyzer := cron.NewJobAnalyzer(db) + analyzer := internal.NewJobAnalyzer(db) if err := analyzer.AnalyzeJobs(); err != nil { log.Fatalf("Error analyzing jobs: %v", err) diff --git a/apps/cron-analyzer/go.mod b/apps/cron-analyzer/go.mod new file mode 100644 index 0000000..ec40919 --- /dev/null +++ b/apps/cron-analyzer/go.mod @@ -0,0 +1,29 @@ +module github.com/jobs-scraper/apps/cron-analyzer + +go 1.24.0 + +require ( + github.com/jobs-scraper/internal/pkg/domain v0.0.0 + github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0-00010101000000-000000000000 + github.com/jobs-scraper/internal/pkg/openrouter v0.0.0-00010101000000-000000000000 + github.com/jobs-scraper/libs/repo v0.0.0 + github.com/joho/godotenv v1.5.1 +) + +require ( + github.com/eduardolat/openroutergo v0.1.0 // indirect + github.com/golang-migrate/migrate/v4 v4.19.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/lib/pq v1.10.9 // indirect + github.com/orsinium-labs/enum v1.4.0 // indirect + github.com/rabbitmq/amqp091-go v1.10.0 // indirect +) + +replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain + +replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure + +replace github.com/jobs-scraper/internal/pkg/openrouter => ../../internal/pkg/openrouter + +replace github.com/jobs-scraper/libs/repo => ../../libs/repo diff --git a/apps/cron-analyzer/go.sum b/apps/cron-analyzer/go.sum new file mode 100644 index 0000000..e1b7cd6 --- /dev/null +++ b/apps/cron-analyzer/go.sum @@ -0,0 +1,79 @@ +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4= +github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/eduardolat/openroutergo v0.1.0 h1:ZD5pG0emgICeHKC4KCtEDD2BIW2LdhDKa2KMbbhrSxI= +github.com/eduardolat/openroutergo v0.1.0/go.mod h1:JVthRi3X9+DtJobL0QFeRqGdCYFj+02fJKbxEngaGAY= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE= +github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/matryer/is v1.4.1 h1:55ehd8zaGABKLXQUe2awZ99BD/PTc2ls+KV/dXphgEQ= +github.com/matryer/is v1.4.1/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/orsinium-labs/enum v1.4.0 h1:3NInlfV76kuAg0kq2FFUondmg3WO7gMEgrPPrlzLDUM= +github.com/orsinium-labs/enum v1.4.0/go.mod h1:Qj5IK2pnElZtkZbGDxZMjpt7SUsn4tqE5vRelmWaBbc= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cron/analyze-jobs.go b/apps/cron-analyzer/internal/analyze-jobs.go similarity index 90% rename from cron/analyze-jobs.go rename to apps/cron-analyzer/internal/analyze-jobs.go index 1b6d1dd..897dcd9 100644 --- a/cron/analyze-jobs.go +++ b/apps/cron-analyzer/internal/analyze-jobs.go @@ -1,4 +1,4 @@ -package cron +package internal import ( "database/sql" @@ -6,10 +6,10 @@ import ( "log" "os" - "github.com/jobs-scraper/shared/domain" - "github.com/jobs-scraper/shared/infrastructure/rabbitmq" - "github.com/jobs-scraper/shared/openrouter" - "github.com/jobs-scraper/shared/repo" + "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" + "github.com/jobs-scraper/internal/pkg/openrouter" + "github.com/jobs-scraper/libs/repo" ) type JobAnalyzer struct { @@ -17,7 +17,7 @@ type JobAnalyzer struct { jobRepo *repo.JobRepository jobDescriptionRepo *repo.JobDescriptionRepository jobAnalysisResultRepo *repo.JobAnalysisResultRepository - openRouterService services.OpenRouterService + openRouterService openrouter.OpenRouterService rmq *rabbitmq.RabbitMQClient } @@ -28,7 +28,7 @@ func NewJobAnalyzer(db *sql.DB) *JobAnalyzer { jobRepo := repo.NewJobRepository(db) jobDescriptionRepo := repo.NewJobDescriptionRepository(db) jobAnalysisResultRepo := repo.NewJobAnalysisResultRepository(db) - openRouterService := services.NewOpenRouterService(model, apiKey) + openRouterService := openrouter.NewOpenRouterService(model, apiKey) rmq, err := rabbitmq.NewRabbitMQClient() if err != nil { @@ -55,7 +55,7 @@ func (ja *JobAnalyzer) AnalyzeJobs() error { log.Printf("Found %d jobs to analyze", len(jobs)) // Read CV file - cv, err := os.ReadFile("cv.txt") + cv, err := os.ReadFile("../../cv.txt") if err != nil { return err } diff --git a/apps/cv-analyzer/bin/cv-analyzer b/apps/cv-analyzer/bin/cv-analyzer new file mode 100755 index 0000000..1ebf925 Binary files /dev/null and b/apps/cv-analyzer/bin/cv-analyzer differ diff --git a/cv/main.go b/apps/cv-analyzer/cmd/cli/main.go similarity index 82% rename from cv/main.go rename to apps/cv-analyzer/cmd/cli/main.go index 81b81b7..d01363d 100644 --- a/cv/main.go +++ b/apps/cv-analyzer/cmd/cli/main.go @@ -4,10 +4,10 @@ import ( "log" "os" - "github.com/jobs-scraper/shared/domain" - "github.com/jobs-scraper/shared/infrastructure" - "github.com/jobs-scraper/shared/openrouter" - "github.com/jobs-scraper/shared/repo" + "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/internal/pkg/infrastructure" + "github.com/jobs-scraper/internal/pkg/openrouter" + "github.com/jobs-scraper/libs/repo" "github.com/joho/godotenv" ) @@ -38,9 +38,9 @@ func main() { jobRepo := repo.NewJobRepository(db) jobDescriptionRepo := repo.NewJobDescriptionRepository(db) - openRouterService := services.NewOpenRouterService(model, apiKey) + openRouterService := openrouter.NewOpenRouterService(model, apiKey) - cv, err := os.ReadFile("../cv.txt") + cv, err := os.ReadFile("../../cv.txt") if err != nil { log.Fatalf("Failed to get cv: %v", err) diff --git a/apps/cv-analyzer/go.mod b/apps/cv-analyzer/go.mod new file mode 100644 index 0000000..47cf47f --- /dev/null +++ b/apps/cv-analyzer/go.mod @@ -0,0 +1,15 @@ +module github.com/jobs-scraper/apps/cv-analyzer + +go 1.24.0 + +require ( + github.com/jobs-scraper/internal/pkg/domain v0.0.0 + github.com/jobs-scraper/libs/repo v0.0.0-00010101000000-000000000000 + github.com/joho/godotenv v1.5.1 +) + +require github.com/lib/pq v1.10.9 // indirect + +replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain + +replace github.com/jobs-scraper/libs/repo => ../../libs/repo diff --git a/apps/cv-analyzer/go.sum b/apps/cv-analyzer/go.sum new file mode 100644 index 0000000..3a912e2 --- /dev/null +++ b/apps/cv-analyzer/go.sum @@ -0,0 +1,2 @@ +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..5b2f033 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,37 @@ +version: '3.8' + +services: + postgres: + image: postgres:15-alpine + environment: + POSTGRES_DB: jobs_scraper + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + ports: + - "5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + + rabbitmq: + image: rabbitmq:3-management-alpine + environment: + RABBITMQ_DEFAULT_USER: guest + RABBITMQ_DEFAULT_PASS: guest + ports: + - "5672:5672" + - "15672:15672" + volumes: + - rabbitmq_data:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "ping"] + interval: 10s + timeout: 5s + retries: 5 +volumes: + postgres_data: + rabbitmq_data: diff --git a/go.mod b/go.mod index d1d59fe..a84c11b 100644 --- a/go.mod +++ b/go.mod @@ -4,16 +4,7 @@ go 1.24.0 toolchain go1.24.7 -require ( - github.com/eduardolat/openroutergo v0.1.0 - github.com/golang-migrate/migrate/v4 v4.19.0 - github.com/gorilla/mux v1.8.1 - github.com/joho/godotenv v1.5.1 - github.com/lib/pq v1.10.9 - github.com/rabbitmq/amqp091-go v1.10.0 - github.com/swaggo/http-swagger v1.3.4 - github.com/swaggo/swag v1.16.3 -) +require github.com/swaggo/swag v1.16.3 require ( github.com/KyleBanks/depth v1.2.1 // indirect @@ -23,14 +14,11 @@ require ( github.com/go-openapi/swag v0.19.15 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/mailru/easyjson v0.7.6 // indirect - github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe // indirect - golang.org/x/tools v0.24.0 // indirect + golang.org/x/tools v0.36.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) require ( - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/orsinium-labs/enum v1.4.0 // indirect - golang.org/x/net v0.44.0 // indirect + github.com/stretchr/testify v1.10.0 // indirect + golang.org/x/sync v0.17.0 // indirect ) diff --git a/go.sum b/go.sum index a161873..72e6649 100644 --- a/go.sum +++ b/go.sum @@ -1,35 +1,9 @@ -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= -github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= -github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= -github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= -github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4= -github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU= -github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= -github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= -github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/eduardolat/openroutergo v0.1.0 h1:ZD5pG0emgICeHKC4KCtEDD2BIW2LdhDKa2KMbbhrSxI= -github.com/eduardolat/openroutergo v0.1.0/go.mod h1:JVthRi3X9+DtJobL0QFeRqGdCYFj+02fJKbxEngaGAY= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -40,19 +14,6 @@ github.com/go-openapi/spec v0.20.6/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE= -github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0= -github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= -github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -60,73 +21,24 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= -github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/matryer/is v1.4.1 h1:55ehd8zaGABKLXQUe2awZ99BD/PTc2ls+KV/dXphgEQ= -github.com/matryer/is v1.4.1/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= -github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= -github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= -github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/orsinium-labs/enum v1.4.0 h1:3NInlfV76kuAg0kq2FFUondmg3WO7gMEgrPPrlzLDUM= -github.com/orsinium-labs/enum v1.4.0/go.mod h1:Qj5IK2pnElZtkZbGDxZMjpt7SUsn4tqE5vRelmWaBbc= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= -github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe h1:K8pHPVoTgxFJt1lXuIzzOX7zZhZFldJQK/CgKx9BFIc= -github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe/go.mod h1:lKJPbtWzJ9JhsTN1k1gZgleJWY/cqq0psdoMmaThG3w= -github.com/swaggo/http-swagger v1.3.4 h1:q7t/XLx0n15H1Q9/tk3Y9L4n210XzJF5WtnDX64a5ww= -github.com/swaggo/http-swagger v1.3.4/go.mod h1:9dAh0unqMBAlbp1uE2Uc2mQTxNMU/ha4UbucIg1MFkQ= github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg= github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= -go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= -golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= -golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= -golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= -golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= +golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= diff --git a/go.work b/go.work index 8e311d5..4b187ab 100644 --- a/go.work +++ b/go.work @@ -2,11 +2,17 @@ go 1.24.0 use ( . - ./packages/linked-scraper - ./shared/domain - ./shared/infrastructure - ./shared/ports - ./shared/repo - ./shared/server - ./shared/utils + ./apps/cron-analyzer + ./apps/cv-analyzer + ./internal/pkg/domain + ./internal/pkg/infrastructure + ./internal/pkg/openrouter + ./internal/pkg/utils + ./internal/swagger + ./libs/ports + ./libs/repo + ./libs/server + ./services/api + ./services/scraper-linkedin + ./tools ) diff --git a/migrations/001_jobs.down.sql b/internal/migrations/001_jobs.down.sql similarity index 100% rename from migrations/001_jobs.down.sql rename to internal/migrations/001_jobs.down.sql diff --git a/migrations/001_jobs.up.sql b/internal/migrations/001_jobs.up.sql similarity index 100% rename from migrations/001_jobs.up.sql rename to internal/migrations/001_jobs.up.sql diff --git a/migrations/002_job_descriptions.down.sql b/internal/migrations/002_job_descriptions.down.sql similarity index 100% rename from migrations/002_job_descriptions.down.sql rename to internal/migrations/002_job_descriptions.down.sql diff --git a/migrations/002_job_descriptions.up.sql b/internal/migrations/002_job_descriptions.up.sql similarity index 100% rename from migrations/002_job_descriptions.up.sql rename to internal/migrations/002_job_descriptions.up.sql diff --git a/migrations/003_add_unique_constraint_job_id.down.sql b/internal/migrations/003_add_unique_constraint_job_id.down.sql similarity index 100% rename from migrations/003_add_unique_constraint_job_id.down.sql rename to internal/migrations/003_add_unique_constraint_job_id.down.sql diff --git a/migrations/003_add_unique_constraint_job_id.up.sql b/internal/migrations/003_add_unique_constraint_job_id.up.sql similarity index 100% rename from migrations/003_add_unique_constraint_job_id.up.sql rename to internal/migrations/003_add_unique_constraint_job_id.up.sql diff --git a/migrations/004_job_link_unique.down.sql b/internal/migrations/004_job_link_unique.down.sql similarity index 100% rename from migrations/004_job_link_unique.down.sql rename to internal/migrations/004_job_link_unique.down.sql diff --git a/migrations/004_job_link_unique.up.sql b/internal/migrations/004_job_link_unique.up.sql similarity index 100% rename from migrations/004_job_link_unique.up.sql rename to internal/migrations/004_job_link_unique.up.sql diff --git a/migrations/005_job_provider.down.sql b/internal/migrations/005_job_provider.down.sql similarity index 100% rename from migrations/005_job_provider.down.sql rename to internal/migrations/005_job_provider.down.sql diff --git a/migrations/005_job_provider.up.sql b/internal/migrations/005_job_provider.up.sql similarity index 100% rename from migrations/005_job_provider.up.sql rename to internal/migrations/005_job_provider.up.sql diff --git a/migrations/006_job_timestamp.down.sql b/internal/migrations/006_job_timestamp.down.sql similarity index 100% rename from migrations/006_job_timestamp.down.sql rename to internal/migrations/006_job_timestamp.down.sql diff --git a/migrations/006_job_timestamp.up.sql b/internal/migrations/006_job_timestamp.up.sql similarity index 100% rename from migrations/006_job_timestamp.up.sql rename to internal/migrations/006_job_timestamp.up.sql diff --git a/migrations/007_add_job_status.down.sql b/internal/migrations/007_add_job_status.down.sql similarity index 100% rename from migrations/007_add_job_status.down.sql rename to internal/migrations/007_add_job_status.down.sql diff --git a/migrations/007_add_job_status.up.sql b/internal/migrations/007_add_job_status.up.sql similarity index 100% rename from migrations/007_add_job_status.up.sql rename to internal/migrations/007_add_job_status.up.sql diff --git a/migrations/008_job_analysis_results.down.sql b/internal/migrations/008_job_analysis_results.down.sql similarity index 100% rename from migrations/008_job_analysis_results.down.sql rename to internal/migrations/008_job_analysis_results.down.sql diff --git a/migrations/008_job_analysis_results.up.sql b/internal/migrations/008_job_analysis_results.up.sql similarity index 100% rename from migrations/008_job_analysis_results.up.sql rename to internal/migrations/008_job_analysis_results.up.sql diff --git a/internal/pkg/domain/go.mod b/internal/pkg/domain/go.mod new file mode 100644 index 0000000..5447dc2 --- /dev/null +++ b/internal/pkg/domain/go.mod @@ -0,0 +1,5 @@ +module github.com/jobs-scraper/internal/pkg/domain + +go 1.24.0 + +toolchain go1.24.7 diff --git a/shared/domain/job-description.go b/internal/pkg/domain/job-description.go similarity index 100% rename from shared/domain/job-description.go rename to internal/pkg/domain/job-description.go diff --git a/shared/domain/job-with-description.go b/internal/pkg/domain/job-with-description.go similarity index 100% rename from shared/domain/job-with-description.go rename to internal/pkg/domain/job-with-description.go diff --git a/shared/domain/job.go b/internal/pkg/domain/job.go similarity index 100% rename from shared/domain/job.go rename to internal/pkg/domain/job.go diff --git a/shared/infrastructure/db.go b/internal/pkg/infrastructure/db.go similarity index 93% rename from shared/infrastructure/db.go rename to internal/pkg/infrastructure/db.go index 7657b91..54e9bbf 100644 --- a/shared/infrastructure/db.go +++ b/internal/pkg/infrastructure/db.go @@ -157,11 +157,13 @@ func getMigrationsPath() (string, error) { if !ok { return "", fmt.Errorf("failed to get current file path") } - - // Get the project root (go up from infrastructure/ to project root) - projectRoot := filepath.Dir(filepath.Dir(filename)) - migrationsPath := filepath.Join(projectRoot, "migrations") - + + // Get the internal directory (go up from internal/pkg/infrastructure/ to internal/) + // Current path: .../internal/pkg/infrastructure/db.go + // Need to go up: infrastructure -> pkg -> internal + internalDir := filepath.Dir(filepath.Dir(filepath.Dir(filename))) + migrationsPath := filepath.Join(internalDir, "migrations") + // Verify the migrations directory exists if _, err := os.Stat(migrationsPath); err != nil { return "", fmt.Errorf("migrations directory not found at %s: %w", migrationsPath, err) diff --git a/internal/pkg/infrastructure/go.mod b/internal/pkg/infrastructure/go.mod new file mode 100644 index 0000000..52a2850 --- /dev/null +++ b/internal/pkg/infrastructure/go.mod @@ -0,0 +1,27 @@ +module github.com/jobs-scraper/internal/pkg/infrastructure + +go 1.24.0 + +toolchain go1.24.7 + +require ( + github.com/golang-migrate/migrate/v4 v4.19.0 + github.com/gorilla/mux v1.8.1 + github.com/jobs-scraper/libs/ports v0.0.0 + github.com/jobs-scraper/libs/repo v0.0.0 + github.com/jobs-scraper/internal/pkg/utils v0.0.0 + github.com/lib/pq v1.10.9 + github.com/rabbitmq/amqp091-go v1.10.0 +) + +replace github.com/jobs-scraper/libs/repo => ../../../libs/repo + +replace github.com/jobs-scraper/internal/pkg/utils => ../utils + +replace github.com/jobs-scraper/libs/ports => ../../../libs/ports + +require ( + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + golang.org/x/sys v0.36.0 // indirect +) diff --git a/shared/infrastructure/go.sum b/internal/pkg/infrastructure/go.sum similarity index 100% rename from shared/infrastructure/go.sum rename to internal/pkg/infrastructure/go.sum diff --git a/shared/infrastructure/http/job.go b/internal/pkg/infrastructure/http/job.go similarity index 97% rename from shared/infrastructure/http/job.go rename to internal/pkg/infrastructure/http/job.go index 2304bdc..1dae58b 100644 --- a/shared/infrastructure/http/job.go +++ b/internal/pkg/infrastructure/http/job.go @@ -7,9 +7,9 @@ import ( "net/http" "strconv" - "github.com/jobs-scraper/shared/ports" - "github.com/jobs-scraper/shared/repo" - "github.com/jobs-scraper/shared/utils" + "github.com/jobs-scraper/libs/ports" + "github.com/jobs-scraper/libs/repo" + "github.com/jobs-scraper/internal/pkg/utils" "github.com/gorilla/mux" ) diff --git a/shared/infrastructure/http/middlewares.go b/internal/pkg/infrastructure/http/middlewares.go similarity index 100% rename from shared/infrastructure/http/middlewares.go rename to internal/pkg/infrastructure/http/middlewares.go diff --git a/shared/infrastructure/rabbitmq/messages.go b/internal/pkg/infrastructure/rabbitmq/messages.go similarity index 100% rename from shared/infrastructure/rabbitmq/messages.go rename to internal/pkg/infrastructure/rabbitmq/messages.go diff --git a/shared/infrastructure/rabbitmq/rabbitmq.go b/internal/pkg/infrastructure/rabbitmq/rabbitmq.go similarity index 100% rename from shared/infrastructure/rabbitmq/rabbitmq.go rename to internal/pkg/infrastructure/rabbitmq/rabbitmq.go diff --git a/internal/pkg/openrouter/go.mod b/internal/pkg/openrouter/go.mod new file mode 100644 index 0000000..23a87a4 --- /dev/null +++ b/internal/pkg/openrouter/go.mod @@ -0,0 +1,12 @@ +module github.com/jobs-scraper/internal/pkg/openrouter + +go 1.24.0 + +require ( + github.com/eduardolat/openroutergo v0.1.0 + github.com/jobs-scraper/internal/pkg/domain v0.0.0 +) + +require github.com/orsinium-labs/enum v1.4.0 // indirect + +replace github.com/jobs-scraper/internal/pkg/domain => ../domain diff --git a/internal/pkg/openrouter/go.sum b/internal/pkg/openrouter/go.sum new file mode 100644 index 0000000..9c8e70f --- /dev/null +++ b/internal/pkg/openrouter/go.sum @@ -0,0 +1,6 @@ +github.com/eduardolat/openroutergo v0.1.0 h1:ZD5pG0emgICeHKC4KCtEDD2BIW2LdhDKa2KMbbhrSxI= +github.com/eduardolat/openroutergo v0.1.0/go.mod h1:JVthRi3X9+DtJobL0QFeRqGdCYFj+02fJKbxEngaGAY= +github.com/matryer/is v1.4.1 h1:55ehd8zaGABKLXQUe2awZ99BD/PTc2ls+KV/dXphgEQ= +github.com/matryer/is v1.4.1/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= +github.com/orsinium-labs/enum v1.4.0 h1:3NInlfV76kuAg0kq2FFUondmg3WO7gMEgrPPrlzLDUM= +github.com/orsinium-labs/enum v1.4.0/go.mod h1:Qj5IK2pnElZtkZbGDxZMjpt7SUsn4tqE5vRelmWaBbc= diff --git a/shared/openrouter/openrouter.go b/internal/pkg/openrouter/openrouter.go similarity index 98% rename from shared/openrouter/openrouter.go rename to internal/pkg/openrouter/openrouter.go index 76654fb..147d481 100644 --- a/shared/openrouter/openrouter.go +++ b/internal/pkg/openrouter/openrouter.go @@ -1,4 +1,4 @@ -package services +package openrouter import ( "encoding/json" @@ -7,7 +7,7 @@ import ( "strings" "github.com/eduardolat/openroutergo" - "github.com/jobs-scraper/shared/domain" + "github.com/jobs-scraper/internal/pkg/domain" ) // JobAnalysisResult represents the structured response from job analysis diff --git a/internal/pkg/utils/go.mod b/internal/pkg/utils/go.mod new file mode 100644 index 0000000..1f9e86e --- /dev/null +++ b/internal/pkg/utils/go.mod @@ -0,0 +1,5 @@ +module github.com/jobs-scraper/internal/pkg/utils + +go 1.24.0 + +toolchain go1.24.7 diff --git a/shared/utils/http.go b/internal/pkg/utils/http.go similarity index 100% rename from shared/utils/http.go rename to internal/pkg/utils/http.go diff --git a/shared/utils/retryable-http-request.go b/internal/pkg/utils/retryable-http-request.go similarity index 100% rename from shared/utils/retryable-http-request.go rename to internal/pkg/utils/retryable-http-request.go diff --git a/libs/ports/go.mod b/libs/ports/go.mod new file mode 100644 index 0000000..1bb16ea --- /dev/null +++ b/libs/ports/go.mod @@ -0,0 +1,9 @@ +module github.com/jobs-scraper/libs/ports + +go 1.24.0 + +toolchain go1.24.7 + +require github.com/jobs-scraper/internal/pkg/domain v0.0.0 + +replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain diff --git a/shared/ports/job-commands.go b/libs/ports/job-commands.go similarity index 88% rename from shared/ports/job-commands.go rename to libs/ports/job-commands.go index 70faf40..9a834c0 100644 --- a/shared/ports/job-commands.go +++ b/libs/ports/job-commands.go @@ -1,6 +1,6 @@ package ports -import "github.com/jobs-scraper/shared/domain" +import "github.com/jobs-scraper/internal/pkg/domain" type JobCommands struct { CreateJob JobCommandHandler diff --git a/libs/repo/go.mod b/libs/repo/go.mod new file mode 100644 index 0000000..18e388c --- /dev/null +++ b/libs/repo/go.mod @@ -0,0 +1,12 @@ +module github.com/jobs-scraper/libs/repo + +go 1.24.0 + +toolchain go1.24.7 + +require ( + github.com/jobs-scraper/internal/pkg/domain v0.0.0 + github.com/lib/pq v1.10.9 +) + +replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain diff --git a/shared/repo/go.sum b/libs/repo/go.sum similarity index 100% rename from shared/repo/go.sum rename to libs/repo/go.sum diff --git a/shared/repo/job-analysis-result.go b/libs/repo/job-analysis-result.go similarity index 98% rename from shared/repo/job-analysis-result.go rename to libs/repo/job-analysis-result.go index 5bb5030..8ba0f28 100644 --- a/shared/repo/job-analysis-result.go +++ b/libs/repo/job-analysis-result.go @@ -4,7 +4,7 @@ import ( "database/sql" "fmt" - "github.com/jobs-scraper/shared/domain" + "github.com/jobs-scraper/internal/pkg/domain" "github.com/lib/pq" ) diff --git a/shared/repo/job-description.go b/libs/repo/job-description.go similarity index 97% rename from shared/repo/job-description.go rename to libs/repo/job-description.go index 870051d..f1cbc68 100644 --- a/shared/repo/job-description.go +++ b/libs/repo/job-description.go @@ -6,7 +6,7 @@ import ( "fmt" "strings" - "github.com/jobs-scraper/shared/domain" + "github.com/jobs-scraper/internal/pkg/domain" ) type JobDescriptionRepository struct { diff --git a/shared/repo/job.go b/libs/repo/job.go similarity index 98% rename from shared/repo/job.go rename to libs/repo/job.go index c57a48b..576a4a9 100644 --- a/shared/repo/job.go +++ b/libs/repo/job.go @@ -6,7 +6,7 @@ import ( "fmt" "strconv" - "github.com/jobs-scraper/shared/domain" + "github.com/jobs-scraper/internal/pkg/domain" ) type JobRepository struct { diff --git a/libs/server/go.mod b/libs/server/go.mod new file mode 100644 index 0000000..0108d78 --- /dev/null +++ b/libs/server/go.mod @@ -0,0 +1,5 @@ +module github.com/jobs-scraper/libs/server + +go 1.24.0 + +require github.com/gorilla/mux v1.8.1 diff --git a/shared/server/go.sum b/libs/server/go.sum similarity index 100% rename from shared/server/go.sum rename to libs/server/go.sum diff --git a/shared/server/server.go b/libs/server/server.go similarity index 100% rename from shared/server/server.go rename to libs/server/server.go diff --git a/packages/glassdoor-scraper/.env.example b/packages/glassdoor-scraper/.env.example deleted file mode 100644 index 49f44a9..0000000 --- a/packages/glassdoor-scraper/.env.example +++ /dev/null @@ -1,10 +0,0 @@ -# RabbitMQ Configuration -RABBITMQ_URL=amqp://localhost:5672 -GLASSDOOR_QUEUE_NAME=scraper.glassdoor -SCRAPER_EXCHANGE_NAME=scraper_exchange - -# Chrome Configuration (optional) -CHROME_PATH=/Applications/Google Chrome.app/Contents/MacOS/Google Chrome - -# Scraping Configuration -USER_AGENT=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 diff --git a/packages/linked-scraper/.env b/packages/linked-scraper/.env deleted file mode 100644 index eb0e8aa..0000000 --- a/packages/linked-scraper/.env +++ /dev/null @@ -1,19 +0,0 @@ -DB_HOST=localhost -DB_PORT=5432 -DB_USER=postgres -DB_PASSWORD=password -DB_NAME=linkedin_jobs -DB_SSLMODE=disable - -SERVER_PORT=8080 -SERVER_HOST=localhost - -CV_AI_MODEL=alibaba/tongyi-deepresearch-30b-a3b:free -OPENROUTER_API_KEY=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 - -#deepseek/deepseek-chat-v3.1:free -# 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 -# qwen/qwen3-235b-a22b:free - - -# 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 \ No newline at end of file diff --git a/scripts/docker/docker-compose.yml b/scripts/docker/docker-compose.yml new file mode 100644 index 0000000..5b2f033 --- /dev/null +++ b/scripts/docker/docker-compose.yml @@ -0,0 +1,37 @@ +version: '3.8' + +services: + postgres: + image: postgres:15-alpine + environment: + POSTGRES_DB: jobs_scraper + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + ports: + - "5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + + rabbitmq: + image: rabbitmq:3-management-alpine + environment: + RABBITMQ_DEFAULT_USER: guest + RABBITMQ_DEFAULT_PASS: guest + ports: + - "5672:5672" + - "15672:15672" + volumes: + - rabbitmq_data:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "ping"] + interval: 10s + timeout: 5s + retries: 5 +volumes: + postgres_data: + rabbitmq_data: diff --git a/services/api/bin/api b/services/api/bin/api new file mode 100755 index 0000000..3dd3e37 Binary files /dev/null and b/services/api/bin/api differ diff --git a/packages/api/main.go b/services/api/cmd/server/main.go similarity index 86% rename from packages/api/main.go rename to services/api/cmd/server/main.go index e5816ab..0813f8b 100644 --- a/packages/api/main.go +++ b/services/api/cmd/server/main.go @@ -10,12 +10,12 @@ import ( "time" "github.com/gorilla/mux" - "github.com/jobs-scraper/packages/api/app" - "github.com/jobs-scraper/shared/infrastructure" - httpHandler "github.com/jobs-scraper/shared/infrastructure/http" - "github.com/jobs-scraper/shared/infrastructure/rabbitmq" - "github.com/jobs-scraper/shared/repo" - _ "github.com/jobs-scraper/swagger" + "github.com/jobs-scraper/services/api/internal/app" + httpHandler "github.com/jobs-scraper/services/api/pkg/http" + "github.com/jobs-scraper/internal/pkg/infrastructure" + "github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" + "github.com/jobs-scraper/libs/repo" + _ "github.com/jobs-scraper/services/api/pkg/swagger" "github.com/joho/godotenv" httpSwagger "github.com/swaggo/http-swagger" ) diff --git a/services/api/go.mod b/services/api/go.mod new file mode 100644 index 0000000..1c207ab --- /dev/null +++ b/services/api/go.mod @@ -0,0 +1,47 @@ +module github.com/jobs-scraper/services/api + +go 1.24.0 + +require ( + github.com/gorilla/mux v1.8.1 + github.com/jobs-scraper/internal/pkg/domain v0.0.0 + github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0-00010101000000-000000000000 + github.com/jobs-scraper/internal/pkg/utils v0.0.0 + github.com/jobs-scraper/libs/ports v0.0.0 + github.com/jobs-scraper/libs/repo v0.0.0 + github.com/jobs-scraper/libs/server v0.0.0-00010101000000-000000000000 + github.com/joho/godotenv v1.5.1 + github.com/swaggo/http-swagger v1.3.4 + github.com/swaggo/swag v1.16.3 +) + +require ( + github.com/KyleBanks/depth v1.2.1 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.20.0 // indirect + github.com/go-openapi/spec v0.20.6 // indirect + github.com/go-openapi/swag v0.19.15 // indirect + github.com/golang-migrate/migrate/v4 v4.19.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/lib/pq v1.10.9 // indirect + github.com/mailru/easyjson v0.7.6 // indirect + github.com/rabbitmq/amqp091-go v1.10.0 // indirect + github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/tools v0.24.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) + +replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain + +replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure + +replace github.com/jobs-scraper/internal/pkg/utils => ../../internal/pkg/utils + +replace github.com/jobs-scraper/libs/ports => ../../libs/ports + +replace github.com/jobs-scraper/libs/repo => ../../libs/repo + +replace github.com/jobs-scraper/libs/server => ../../libs/server diff --git a/services/api/go.sum b/services/api/go.sum new file mode 100644 index 0000000..8512564 --- /dev/null +++ b/services/api/go.sum @@ -0,0 +1,134 @@ +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= +github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4= +github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= +github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= +github.com/go-openapi/spec v0.20.6 h1:ich1RQ3WDbfoeTqTAb+5EIxNmpKVJZWBNah9RAT0jIQ= +github.com/go-openapi/spec v0.20.6/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE= +github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe h1:K8pHPVoTgxFJt1lXuIzzOX7zZhZFldJQK/CgKx9BFIc= +github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe/go.mod h1:lKJPbtWzJ9JhsTN1k1gZgleJWY/cqq0psdoMmaThG3w= +github.com/swaggo/http-swagger v1.3.4 h1:q7t/XLx0n15H1Q9/tk3Y9L4n210XzJF5WtnDX64a5ww= +github.com/swaggo/http-swagger v1.3.4/go.mod h1:9dAh0unqMBAlbp1uE2Uc2mQTxNMU/ha4UbucIg1MFkQ= +github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg= +github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/packages/api/app/app.go b/services/api/internal/app/app.go similarity index 69% rename from packages/api/app/app.go rename to services/api/internal/app/app.go index 19765fe..4b7e9ed 100644 --- a/packages/api/app/app.go +++ b/services/api/internal/app/app.go @@ -4,11 +4,11 @@ import ( "database/sql" "github.com/gorilla/mux" - "github.com/jobs-scraper/packages/api/commands/job" - "github.com/jobs-scraper/shared/infrastructure/rabbitmq" - "github.com/jobs-scraper/shared/ports" - "github.com/jobs-scraper/shared/repo" - "github.com/jobs-scraper/shared/server" + "github.com/jobs-scraper/services/api/internal/commands/job" + "github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" + "github.com/jobs-scraper/libs/ports" + "github.com/jobs-scraper/libs/repo" + "github.com/jobs-scraper/libs/server" ) type Application struct { diff --git a/packages/api/commands/job/create-job.go b/services/api/internal/commands/job/create-job.go similarity index 88% rename from packages/api/commands/job/create-job.go rename to services/api/internal/commands/job/create-job.go index 1b6dc2f..d0a1bb7 100644 --- a/packages/api/commands/job/create-job.go +++ b/services/api/internal/commands/job/create-job.go @@ -5,10 +5,10 @@ import ( "errors" "log" - "github.com/jobs-scraper/shared/domain" - "github.com/jobs-scraper/shared/infrastructure/rabbitmq" - "github.com/jobs-scraper/shared/ports" - "github.com/jobs-scraper/shared/repo" + "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" + "github.com/jobs-scraper/libs/ports" + "github.com/jobs-scraper/libs/repo" ) type CreateJob struct { diff --git a/services/api/pkg/http/job.go b/services/api/pkg/http/job.go new file mode 100644 index 0000000..1dae58b --- /dev/null +++ b/services/api/pkg/http/job.go @@ -0,0 +1,148 @@ +package http + +import ( + "encoding/json" + "fmt" + "log/slog" + "net/http" + "strconv" + + "github.com/jobs-scraper/libs/ports" + "github.com/jobs-scraper/libs/repo" + "github.com/jobs-scraper/internal/pkg/utils" + + "github.com/gorilla/mux" +) + +// JobHandler handles HTTP requests related to jobs +type JobHandler struct { + jobCommands *ports.JobCommands + jobAnalysisResultRepo *repo.JobAnalysisResultRepository +} + +// NewJobHandler creates a new instance of JobHandler +func NewJobHandler(jobCommands *ports.JobCommands, jobAnalysisResultRepo *repo.JobAnalysisResultRepository) *JobHandler { + return &JobHandler{ + jobCommands: jobCommands, + jobAnalysisResultRepo: jobAnalysisResultRepo, + } +} + +// RegisterRoutes registers all routes to the router +func (h *JobHandler) RegisterRoutes(router *mux.Router) { + jobs := router.PathPrefix("/jobs").Subrouter() + + //jobs.Use(AuthMiddleware) + + jobs.HandleFunc("", utils.Make(h.CreateJob)).Methods("POST") + jobs.HandleFunc("/analysis", utils.Make(h.GetAllAnalysisResults)).Methods("GET") + jobs.HandleFunc("/{id}/analysis", utils.Make(h.GetJobAnalysisResult)).Methods("GET") + jobs.HandleFunc("/analysis/top-matches", utils.Make(h.GetTopMatches)).Methods("GET") +} + +// CreateJob handles the creation of a new job +// @Summary Create a new job +// @Description Creates a new job with the provided specifications +// @Tags jobs +// @Accept json +// @Produce json +// @Param job body ports.CreateJobCommand true "Job creation data" +// @Success 201 {object} map[string]string "Successfully created job" +// @Failure 400 {object} map[string]string "Invalid request data" +// @Failure 500 {object} map[string]string "Internal server error" +// @Router /jobs [post] +func (h *JobHandler) CreateJob(w http.ResponseWriter, r *http.Request) error { + var cmd ports.CreateJobCommand + if err := json.NewDecoder(r.Body).Decode(&cmd); err != nil { + slog.Error(err.Error()) + return utils.NewAPIError(http.StatusBadRequest, utils.InvalidJSON()) + } + + err := h.jobCommands.CreateJob.Handle(cmd) + if err != nil { + slog.Error(err.Error()) + return utils.NewAPIError(http.StatusInternalServerError, err) + } + + utils.WriteJSON(w, http.StatusCreated, map[string]string{"message": "Success"}) + return nil +} + +// GetAllAnalysisResults handles retrieving all job analysis results +// @Summary Get all job analysis results +// @Description Retrieves all job analysis results from the database +// @Tags jobs +// @Produce json +// @Success 200 {array} domain.JobAnalysisResult "List of job analysis results" +// @Failure 500 {object} map[string]string "Internal server error" +// @Router /jobs/analysis [get] +func (h *JobHandler) GetAllAnalysisResults(w http.ResponseWriter, r *http.Request) error { + results, err := h.jobAnalysisResultRepo.GetAllAnalysisResults() + if err != nil { + slog.Error("Failed to get analysis results", "error", err) + return utils.NewAPIError(http.StatusInternalServerError, err) + } + + utils.WriteJSON(w, http.StatusOK, results) + return nil +} + +// GetJobAnalysisResult handles retrieving analysis result for a specific job +// @Summary Get job analysis result by job ID +// @Description Retrieves the analysis result for a specific job +// @Tags jobs +// @Produce json +// @Param id path int true "Job ID" +// @Success 200 {object} domain.JobAnalysisResult "Job analysis result" +// @Failure 400 {object} map[string]string "Invalid job ID" +// @Failure 404 {object} map[string]string "Analysis result not found" +// @Failure 500 {object} map[string]string "Internal server error" +// @Router /jobs/{id}/analysis [get] +func (h *JobHandler) GetJobAnalysisResult(w http.ResponseWriter, r *http.Request) error { + vars := mux.Vars(r) + jobID, err := strconv.ParseInt(vars["id"], 10, 64) + if err != nil { + return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid job ID")) + } + + result, err := h.jobAnalysisResultRepo.GetAnalysisResultByJobID(jobID) + if err != nil { + slog.Error("Failed to get analysis result", "jobID", jobID, "error", err) + return utils.NewAPIError(http.StatusNotFound, fmt.Errorf("analysis result not found")) + } + + utils.WriteJSON(w, http.StatusOK, result) + return nil +} + +// GetTopMatches handles retrieving top matching jobs based on analysis score +// @Summary Get top matching jobs +// @Description Retrieves jobs with high analysis match scores +// @Tags jobs +// @Produce json +// @Param min_score query int false "Minimum match score (default: 70)" +// @Success 200 {array} domain.JobAnalysisResult "List of top matching job analysis results" +// @Failure 400 {object} map[string]string "Invalid minimum score" +// @Failure 500 {object} map[string]string "Internal server error" +// @Router /jobs/analysis/top-matches [get] +func (h *JobHandler) GetTopMatches(w http.ResponseWriter, r *http.Request) error { + minScoreStr := r.URL.Query().Get("min_score") + minScore := 70 // default minimum score + + if minScoreStr != "" { + var err error + minScore, err = strconv.Atoi(minScoreStr) + if err != nil { + return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid minimum score")) + } + } + + results, err := h.jobAnalysisResultRepo.GetAnalysisResultsByMatchScore(minScore) + if err != nil { + slog.Error("Failed to get top matches", "minScore", minScore, "error", err) + return utils.NewAPIError(http.StatusInternalServerError, err) + } + + utils.WriteJSON(w, http.StatusOK, results) + return nil +} diff --git a/services/api/pkg/http/middlewares.go b/services/api/pkg/http/middlewares.go new file mode 100644 index 0000000..3b6e0da --- /dev/null +++ b/services/api/pkg/http/middlewares.go @@ -0,0 +1,143 @@ +package http + +import ( + "context" + "encoding/json" + "log" + "net/http" + "strings" + "time" +) + +// CORSMiddleware handles Cross-Origin Resource Sharing +func CORSMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Set CORS headers + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Requested-With") + w.Header().Set("Access-Control-Allow-Credentials", "true") + w.Header().Set("Access-Control-Max-Age", "86400") + + // Handle preflight OPTIONS requests + if r.Method == "OPTIONS" { + w.WriteHeader(http.StatusOK) + return + } + + next.ServeHTTP(w, r) + }) +} + +// LogsMiddleware logs HTTP requests with timing information +func LogsMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + next.ServeHTTP(w, r) + log.Printf("%s %s %s", r.Method, r.RequestURI, time.Since(start)) + }) +} + +// AuthMiddleware handles JWT token validation, API keys, and basic auth +func AuthMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Skip auth for swagger endpoints and health checks + if strings.HasPrefix(r.URL.Path, "/swagger/") || r.URL.Path == "/health" { + next.ServeHTTP(w, r) + return + } + + // Try Bearer token authentication first + authHeader := r.Header.Get("Authorization") + if authHeader != "" { + if strings.HasPrefix(authHeader, "Bearer ") { + token := strings.TrimPrefix(authHeader, "Bearer ") + if token != "" { + userID, err := validateJWTToken(token) + if err == nil { + ctx := context.WithValue(r.Context(), "user_id", userID) + next.ServeHTTP(w, r.WithContext(ctx)) + return + } + } + } + } + + // Try API Key authentication + apiKey := r.Header.Get("X-API-Key") + if apiKey != "" { + if isValidAPIKey(apiKey) { + ctx := context.WithValue(r.Context(), "user_id", getUserIDFromAPIKey(apiKey)) + next.ServeHTTP(w, r.WithContext(ctx)) + return + } + } + + // Try Basic authentication + username, password, ok := r.BasicAuth() + if ok { + if isValidCredentials(username, password) { + ctx := context.WithValue(r.Context(), "username", username) + next.ServeHTTP(w, r.WithContext(ctx)) + return + } + } + + // Fallback to cookie-based auth (existing behavior) + token, err := r.Cookie("token") + if err == nil && token.Value == "123" { + next.ServeHTTP(w, r) + return + } + + // No valid authentication found + response := map[string]interface{}{ + "error": "Unauthorized", + "message": "Valid authentication required. Use Bearer token, API key, or basic auth.", + "code": http.StatusUnauthorized, + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnauthorized) + json.NewEncoder(w).Encode(response) + }) +} + +// Helper functions for authentication validation +func validateJWTToken(token string) (string, error) { + // TODO: Implement JWT validation logic with your preferred JWT library + // For demonstration, accept a demo token + if token == "valid-jwt-token" { + return "user-123", nil + } + return "", http.ErrAbortHandler +} + +func isValidAPIKey(apiKey string) bool { + // TODO: Implement API key validation logic + // For demonstration, accept a demo API key + return apiKey == "demo-api-key-123" +} + +func getUserIDFromAPIKey(apiKey string) string { + // TODO: Implement logic to get user ID from API key + // For demonstration, return a mock user ID + return "api-user-123" +} + +func isValidCredentials(username, password string) bool { + // TODO: Implement credential validation logic + // For demonstration, accept demo credentials + return username == "admin" && password == "password" +} + +// GetUserIDFromContext extracts user ID from request context +func GetUserIDFromContext(ctx context.Context) (string, bool) { + userID, ok := ctx.Value("user_id").(string) + return userID, ok +} + +// GetUsernameFromContext extracts username from request context +func GetUsernameFromContext(ctx context.Context) (string, bool) { + username, ok := ctx.Value("username").(string) + return username, ok +} diff --git a/swagger/docs.go b/services/api/pkg/swagger/docs.go similarity index 100% rename from swagger/docs.go rename to services/api/pkg/swagger/docs.go diff --git a/swagger/swagger.json b/services/api/pkg/swagger/swagger.json similarity index 100% rename from swagger/swagger.json rename to services/api/pkg/swagger/swagger.json diff --git a/swagger/swagger.yaml b/services/api/pkg/swagger/swagger.yaml similarity index 100% rename from swagger/swagger.yaml rename to services/api/pkg/swagger/swagger.yaml diff --git a/packages/glassdoor-scraper/index.ts b/services/scraper-glassdoor/index.ts similarity index 100% rename from packages/glassdoor-scraper/index.ts rename to services/scraper-glassdoor/index.ts diff --git a/packages/glassdoor-scraper/package-lock.json b/services/scraper-glassdoor/package-lock.json similarity index 100% rename from packages/glassdoor-scraper/package-lock.json rename to services/scraper-glassdoor/package-lock.json diff --git a/packages/glassdoor-scraper/package.json b/services/scraper-glassdoor/package.json similarity index 100% rename from packages/glassdoor-scraper/package.json rename to services/scraper-glassdoor/package.json diff --git a/packages/glassdoor-scraper/services/browserService.ts b/services/scraper-glassdoor/services/browserService.ts similarity index 100% rename from packages/glassdoor-scraper/services/browserService.ts rename to services/scraper-glassdoor/services/browserService.ts diff --git a/packages/glassdoor-scraper/tsconfig.json b/services/scraper-glassdoor/tsconfig.json similarity index 100% rename from packages/glassdoor-scraper/tsconfig.json rename to services/scraper-glassdoor/tsconfig.json diff --git a/packages/glassdoor-scraper/utils/browser.ts b/services/scraper-glassdoor/utils/browser.ts similarity index 100% rename from packages/glassdoor-scraper/utils/browser.ts rename to services/scraper-glassdoor/utils/browser.ts diff --git a/packages/glassdoor-scraper/utils/rabbitmq.ts b/services/scraper-glassdoor/utils/rabbitmq.ts similarity index 100% rename from packages/glassdoor-scraper/utils/rabbitmq.ts rename to services/scraper-glassdoor/utils/rabbitmq.ts diff --git a/services/scraper-linkedin/bin/scraper b/services/scraper-linkedin/bin/scraper new file mode 100755 index 0000000..537c910 Binary files /dev/null and b/services/scraper-linkedin/bin/scraper differ diff --git a/packages/linked-scraper/go.mod b/services/scraper-linkedin/go.mod similarity index 50% rename from packages/linked-scraper/go.mod rename to services/scraper-linkedin/go.mod index 59bddf8..f9e8ef3 100644 --- a/packages/linked-scraper/go.mod +++ b/services/scraper-linkedin/go.mod @@ -6,10 +6,10 @@ toolchain go1.24.7 require ( github.com/PuerkitoBio/goquery v1.10.3 - github.com/jobs-scraper/shared/domain v0.0.0 - github.com/jobs-scraper/shared/infrastructure v0.0.0 - github.com/jobs-scraper/shared/repo v0.0.0 - github.com/jobs-scraper/shared/utils v0.0.0 + github.com/jobs-scraper/internal/pkg/domain v0.0.0 + github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0 + github.com/jobs-scraper/internal/pkg/utils v0.0.0 + github.com/jobs-scraper/libs/repo v0.0.0 github.com/joho/godotenv v1.5.1 ) @@ -23,10 +23,10 @@ require ( golang.org/x/net v0.44.0 // indirect ) -replace github.com/jobs-scraper/shared/domain => ../../shared/domain +replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain -replace github.com/jobs-scraper/shared/infrastructure => ../../shared/infrastructure +replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure -replace github.com/jobs-scraper/shared/repo => ../../shared/repo +replace github.com/jobs-scraper/libs/repo => ../../libs/repo -replace github.com/jobs-scraper/shared/utils => ../../shared/utils +replace github.com/jobs-scraper/internal/pkg/utils => ../../internal/pkg/utils diff --git a/packages/linked-scraper/go.sum b/services/scraper-linkedin/go.sum similarity index 100% rename from packages/linked-scraper/go.sum rename to services/scraper-linkedin/go.sum diff --git a/packages/linked-scraper/main.go b/services/scraper-linkedin/main.go similarity index 91% rename from packages/linked-scraper/main.go rename to services/scraper-linkedin/main.go index d4e4b2f..0f34658 100644 --- a/packages/linked-scraper/main.go +++ b/services/scraper-linkedin/main.go @@ -11,10 +11,10 @@ import ( "time" "github.com/jobs-scraper/services/scraper/pipeline" - "github.com/jobs-scraper/shared/domain" - "github.com/jobs-scraper/shared/infrastructure" - "github.com/jobs-scraper/shared/infrastructure/rabbitmq" - "github.com/jobs-scraper/shared/repo" + "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/internal/pkg/infrastructure" + "github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" + "github.com/jobs-scraper/libs/repo" "github.com/joho/godotenv" ) diff --git a/packages/linked-scraper/pipeline/job_pipeline.go b/services/scraper-linkedin/pipeline/job_pipeline.go similarity index 95% rename from packages/linked-scraper/pipeline/job_pipeline.go rename to services/scraper-linkedin/pipeline/job_pipeline.go index 2b6b5c6..bdbb779 100644 --- a/packages/linked-scraper/pipeline/job_pipeline.go +++ b/services/scraper-linkedin/pipeline/job_pipeline.go @@ -9,8 +9,8 @@ import ( "time" - "github.com/jobs-scraper/shared/domain" - "github.com/jobs-scraper/shared/repo" + "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/libs/repo" ) // JobDescriptionResult represents the result of job description scraping diff --git a/packages/linked-scraper/pipeline/job_pipeline_workers.go b/services/scraper-linkedin/pipeline/job_pipeline_workers.go similarity index 97% rename from packages/linked-scraper/pipeline/job_pipeline_workers.go rename to services/scraper-linkedin/pipeline/job_pipeline_workers.go index 4ba09bf..5b39bc0 100644 --- a/packages/linked-scraper/pipeline/job_pipeline_workers.go +++ b/services/scraper-linkedin/pipeline/job_pipeline_workers.go @@ -4,7 +4,7 @@ import ( "context" "log" - "github.com/jobs-scraper/shared/domain" + "github.com/jobs-scraper/internal/pkg/domain" ) func GetJobs(context context.Context, scraperService *Scraper, searchQuery domain.SearchQuery) <-chan domain.Job { diff --git a/packages/linked-scraper/pipeline/scraper.go b/services/scraper-linkedin/pipeline/scraper.go similarity index 98% rename from packages/linked-scraper/pipeline/scraper.go rename to services/scraper-linkedin/pipeline/scraper.go index 9fb803c..97f6b2f 100644 --- a/packages/linked-scraper/pipeline/scraper.go +++ b/services/scraper-linkedin/pipeline/scraper.go @@ -12,8 +12,8 @@ import ( "time" "github.com/PuerkitoBio/goquery" - "github.com/jobs-scraper/shared/domain" - "github.com/jobs-scraper/shared/utils" + "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/internal/pkg/utils" ) type Config struct { diff --git a/packages/linked-scraper/scraper b/services/scraper-linkedin/scraper similarity index 100% rename from packages/linked-scraper/scraper rename to services/scraper-linkedin/scraper diff --git a/shared/domain/go.mod b/shared/domain/go.mod deleted file mode 100644 index 6adfe56..0000000 --- a/shared/domain/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module github.com/jobs-scraper/shared/domain - -go 1.24.0 - -toolchain go1.24.7 diff --git a/shared/infrastructure/go.mod b/shared/infrastructure/go.mod deleted file mode 100644 index 08be7ee..0000000 --- a/shared/infrastructure/go.mod +++ /dev/null @@ -1,27 +0,0 @@ -module github.com/jobs-scraper/shared/infrastructure - -go 1.24.0 - -toolchain go1.24.7 - -require ( - github.com/golang-migrate/migrate/v4 v4.19.0 - github.com/gorilla/mux v1.8.1 - github.com/jobs-scraper/shared/ports v0.0.0 - github.com/jobs-scraper/shared/repo v0.0.0 - github.com/jobs-scraper/shared/utils v0.0.0 - github.com/lib/pq v1.10.9 - github.com/rabbitmq/amqp091-go v1.10.0 -) - -replace github.com/jobs-scraper/shared/repo => ../repo - -replace github.com/jobs-scraper/shared/utils => ../utils - -replace github.com/jobs-scraper/shared/ports => ../ports - -require ( - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect - golang.org/x/sys v0.36.0 // indirect -) diff --git a/shared/ports/go.mod b/shared/ports/go.mod deleted file mode 100644 index e30a1e9..0000000 --- a/shared/ports/go.mod +++ /dev/null @@ -1,9 +0,0 @@ -module github.com/jobs-scraper/shared/ports - -go 1.24.0 - -toolchain go1.24.7 - -require github.com/jobs-scraper/shared/domain v0.0.0 - -replace github.com/jobs-scraper/shared/domain => ../domain diff --git a/shared/repo/go.mod b/shared/repo/go.mod deleted file mode 100644 index c191949..0000000 --- a/shared/repo/go.mod +++ /dev/null @@ -1,12 +0,0 @@ -module github.com/jobs-scraper/shared/repo - -go 1.24.0 - -toolchain go1.24.7 - -require ( - github.com/jobs-scraper/shared/domain v0.0.0 - github.com/lib/pq v1.10.9 -) - -replace github.com/jobs-scraper/shared/domain => ../domain diff --git a/shared/server/go.mod b/shared/server/go.mod deleted file mode 100644 index 1f3a369..0000000 --- a/shared/server/go.mod +++ /dev/null @@ -1,7 +0,0 @@ -module github.com/jobs-scraper/shared/server - -go 1.24.0 - -require github.com/gorilla/mux v1.8.1 - -replace github.com/jobs-scraper/shared/server => ./ diff --git a/shared/utils/go.mod b/shared/utils/go.mod deleted file mode 100644 index b7b0c4a..0000000 --- a/shared/utils/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module github.com/jobs-scraper/shared/utils - -go 1.24.0 - -toolchain go1.24.7 diff --git a/tools/go.mod b/tools/go.mod new file mode 100644 index 0000000..73ef739 --- /dev/null +++ b/tools/go.mod @@ -0,0 +1,7 @@ +module github.com/jobs-scraper/tools + +go 1.24.0 + +require ( + github.com/swaggo/swag v1.16.3 +)