From 2589c817979aeb03f653d1660492f5c4a7375c44 Mon Sep 17 00:00:00 2001 From: Elshimy Ziad Magdy Taha Date: Mon, 15 Dec 2025 12:17:08 +0500 Subject: [PATCH] deploy --- .gitlab-ci.yml | 23 ++++++++ services/scraper-google/.dockerignore | 47 +++++++++++++++ services/scraper-google/.env | 2 +- services/scraper-google/.env.example | 18 ++++++ services/scraper-google/Dockerfile | 40 +++++++++++++ services/scraper-google/Makefile | 67 ++++++++++++++++++++++ services/scraper-google/docker-compose.yml | 49 ++++++++++++++++ services/scraper-google/main.go | 1 - 8 files changed, 245 insertions(+), 2 deletions(-) create mode 100644 .gitlab-ci.yml create mode 100644 services/scraper-google/.dockerignore create mode 100644 services/scraper-google/.env.example create mode 100644 services/scraper-google/Dockerfile create mode 100644 services/scraper-google/Makefile create mode 100644 services/scraper-google/docker-compose.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..59f05d7 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,23 @@ +stages: + - deploy + +deploy_production: + stage: deploy + image: alpine:3.20 + environment: + name: production + url: https://jobs-scraper.ai-assistant-bot.xyz + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + when: manual + allow_failure: false + before_script: + - apk add --no-cache openssh-client rsync + - mkdir -p ~/.ssh + - echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_rsa + - chmod 600 ~/.ssh/id_rsa + - ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts + script: + - ssh "$DEPLOY_USER@$DEPLOY_HOST" "mkdir -p /opt/apps/jobs-scraper" + - rsync -az --delete --exclude='.git' --exclude='.env' --exclude='.env.*' --exclude='.local.env' ./ "$DEPLOY_USER@$DEPLOY_HOST:/opt/apps/jobs-scraper/" + - ssh "$DEPLOY_USER@$DEPLOY_HOST" "cd /opt/apps/jobs-scraper/services/scraper-google && docker compose up -d --build" diff --git a/services/scraper-google/.dockerignore b/services/scraper-google/.dockerignore new file mode 100644 index 0000000..78a6e29 --- /dev/null +++ b/services/scraper-google/.dockerignore @@ -0,0 +1,47 @@ +# Git +.git +.gitignore + +# Environment files (will be mounted or set via docker-compose) +.local.env +*.env + +# Documentation +*.md +DOCKER_README.md +RECOMMENDED_MODELS.md + +# Development files +.vscode +.idea +*.swp +*.swo +*~ + +# Build artifacts +*.exe +*.dll +*.so +*.dylib +scraper-google + +# Test files +*_test.go +testdata/ + +# Logs +*.log +logs/ + +# Temporary files +tmp/ +temp/ + +# OS files +.DS_Store +Thumbs.db + +# Docker files (not needed in image) +Dockerfile +docker-compose.yml +.dockerignore diff --git a/services/scraper-google/.env b/services/scraper-google/.env index dde4e17..1faafa7 100644 --- a/services/scraper-google/.env +++ b/services/scraper-google/.env @@ -12,5 +12,5 @@ OPENAI_API_KEY=sk-GtdLX9YBOCsBEgBL4rBONA OPENAI_BASE_URL=https://hubai.loe.gg/v1 OPENAI_MODEL=gpt-4o-mini -CV_AI_MODEL=your_ai_model +CV_AI_MODEL=alibaba/tongyi-deepresearch-30b-a3b:free OPENROUTER_API_KEY=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 \ No newline at end of file diff --git a/services/scraper-google/.env.example b/services/scraper-google/.env.example new file mode 100644 index 0000000..35c9f7e --- /dev/null +++ b/services/scraper-google/.env.example @@ -0,0 +1,18 @@ +# Database Configuration +DB_HOST=postgres +DB_PORT=5432 +DB_NAME=jobs_scraper +DB_USER=postgres +DB_PASSWORD=postgres + +# OpenRouter Configuration (for job description extraction) +OPENROUTER_API_KEY=your_openrouter_api_key_here +CV_AI_MODEL=google/gemini-flash-1.5 + +# Optional: OpenAI Configuration (alternative to OpenRouter) +# OPENAI_API_KEY=your_openai_api_key_here +# OPENAI_BASE_URL=https://api.openai.com/v1 + +# Chrome/Chromedp Configuration (usually defaults are fine) +CHROMEDP_DISABLE_GPU=true +CHROMEDP_NO_SANDBOX=true diff --git a/services/scraper-google/Dockerfile b/services/scraper-google/Dockerfile new file mode 100644 index 0000000..3278239 --- /dev/null +++ b/services/scraper-google/Dockerfile @@ -0,0 +1,40 @@ +# Build stage +FROM golang:1.24-alpine AS builder + +# Install build dependencies +RUN apk add --no-cache git + +WORKDIR /build + +# Copy go mod files +COPY go.mod go.sum ./ +RUN go mod download + +# Copy the entire monorepo +COPY . . + +# Build the scraper-google service +WORKDIR /build/services/scraper-google +RUN CGO_ENABLED=0 GOOS=linux go build -o scraper-google . + +# Runtime stage +FROM chromedp/headless-shell:latest + +# Install ca-certificates for HTTPS requests +RUN apt-get update && apt-get install -y \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Copy the binary from builder +COPY --from=builder /build/services/scraper-google/scraper-google . + +# Copy environment files if they exist (optional) +COPY --from=builder /build/services/scraper-google/.env* ./ + +# Expose any ports if needed (not required for this service) +# EXPOSE 8080 + +# Run the scraper +CMD ["./scraper-google"] diff --git a/services/scraper-google/Makefile b/services/scraper-google/Makefile new file mode 100644 index 0000000..919f3b9 --- /dev/null +++ b/services/scraper-google/Makefile @@ -0,0 +1,67 @@ +.PHONY: help build up down logs restart clean rebuild + +help: ## Show this help message + @echo 'Usage: make [target]' + @echo '' + @echo 'Available targets:' + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf " %-15s %s\n", $$1, $$2}' $(MAKEFILE_LIST) + +build: ## Build the Docker images + docker-compose build + +up: ## Start all services + docker-compose up -d + +down: ## Stop all services + docker-compose down + +logs: ## View logs (use 'make logs service=scraper-google' for specific service) + @if [ -z "$(service)" ]; then \ + docker-compose logs -f; \ + else \ + docker-compose logs -f $(service); \ + fi + +restart: ## Restart all services + docker-compose restart + +restart-scraper: ## Restart just the scraper service + docker-compose restart scraper-google + +clean: ## Stop services and remove volumes (WARNING: deletes data) + docker-compose down -v + +rebuild: ## Rebuild and restart services + docker-compose down + docker-compose build --no-cache + docker-compose up -d + +ps: ## Show running containers + docker-compose ps + +stats: ## Show resource usage + docker stats + +exec-db: ## Connect to PostgreSQL database + docker-compose exec postgres psql -U postgres -d jobs_scraper + +backup-db: ## Backup the database + @mkdir -p backups + docker-compose exec postgres pg_dump -U postgres jobs_scraper > backups/backup_$(shell date +%Y%m%d_%H%M%S).sql + @echo "Database backed up to backups/backup_$(shell date +%Y%m%d_%H%M%S).sql" + +restore-db: ## Restore database from backup (use 'make restore-db file=backup.sql') + @if [ -z "$(file)" ]; then \ + echo "Error: Please specify backup file. Usage: make restore-db file=backup.sql"; \ + exit 1; \ + fi + docker-compose exec -T postgres psql -U postgres jobs_scraper < $(file) + +shell-scraper: ## Open shell in scraper container + docker-compose exec scraper-google sh + +dev: ## Run in development mode with live logs + docker-compose up --build + +test: ## Run tests (if you have any) + docker-compose exec scraper-google go test ./... diff --git a/services/scraper-google/docker-compose.yml b/services/scraper-google/docker-compose.yml new file mode 100644 index 0000000..5310f1c --- /dev/null +++ b/services/scraper-google/docker-compose.yml @@ -0,0 +1,49 @@ +version: "3.8" + +services: + scraper-google: + build: + context: ../../ + dockerfile: services/scraper-google/Dockerfile + container_name: jobs-scraper-google + environment: + DB_HOST: postgres + DB_PORT: 5432 + DB_NAME: jobs_scraper + DB_USER: scraper_user + DB_PASSWORD: Cocowawa_12345 + + # OpenRouter Configuration + OPENROUTER_API_KEY: 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 + CV_AI_MODEL: alibaba/tongyi-deepresearch-30b-a3b:free + + # Optional: OpenAI Configuration + OPENAI_API_KEY: sk-GtdLX9YBOCsBEgBL4rBONA + OPENAI_BASE_URL: https://hubai.loe.gg/v1 + OPEN_AI_MODEL: gpt-4o-mini + + # Chrome/Chromedp Configuration + CHROMEDP_DISABLE_GPU: true + CHROMEDP_NO_SANDBOX: true + volumes: + - ./.env:/app/.env:ro + restart: unless-stopped + networks: + - scraper-network + # Add resource limits to prevent runaway chrome instances + deploy: + resources: + limits: + memory: 2G + cpus: "2" + reservations: + memory: 512M + cpus: "0.5" + +volumes: + postgres_data: + driver: local + +networks: + scraper-network: + driver: bridge diff --git a/services/scraper-google/main.go b/services/scraper-google/main.go index 508ec61..8e1e530 100644 --- a/services/scraper-google/main.go +++ b/services/scraper-google/main.go @@ -53,7 +53,6 @@ func main() { log.Fatalf("Failed to start scheduler: %v", err) } - // Process and save jobs immediately as they're scraped go func() { for googleLink := range googleLinkStream { htmlRaw, url, err := utils.GetHTMLRaw(ctx, googleLink)