deploy

.gitlab-ci.yml

@@ -0,0 +1,23 @@
+stages:
+  - deploy
+
+deploy_production:
+  stage: deploy
+  image: alpine:3.20
+  environment:
+    name: production
+    url: https://jobs-scraper.ai-assistant-bot.xyz
+  rules:
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+      when: manual
+      allow_failure: false
+  before_script:
+    - apk add --no-cache openssh-client rsync
+    - mkdir -p ~/.ssh
+    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_rsa
+    - chmod 600 ~/.ssh/id_rsa
+    - ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts
+  script:
+    - ssh "$DEPLOY_USER@$DEPLOY_HOST" "mkdir -p /opt/apps/jobs-scraper"
+    - rsync -az --delete --exclude='.git' --exclude='.env' --exclude='.env.*' --exclude='.local.env' ./ "$DEPLOY_USER@$DEPLOY_HOST:/opt/apps/jobs-scraper/"
+    - ssh "$DEPLOY_USER@$DEPLOY_HOST" "cd /opt/apps/jobs-scraper/services/scraper-google && docker compose up -d --build"

services/scraper-google/.dockerignore

@@ -0,0 +1,47 @@
+# Git
+.git
+.gitignore
+
+# Environment files (will be mounted or set via docker-compose)
+.local.env
+*.env
+
+# Documentation
+*.md
+DOCKER_README.md
+RECOMMENDED_MODELS.md
+
+# Development files
+.vscode
+.idea
+*.swp
+*.swo
+*~
+
+# Build artifacts
+*.exe
+*.dll
+*.so
+*.dylib
+scraper-google
+
+# Test files
+*_test.go
+testdata/
+
+# Logs
+*.log
+logs/
+
+# Temporary files
+tmp/
+temp/
+
+# OS files
+.DS_Store
+Thumbs.db
+
+# Docker files (not needed in image)
+Dockerfile
+docker-compose.yml
+.dockerignore

services/scraper-google/.env

@@ -12,5 +12,5 @@ OPENAI_API_KEY=sk-GtdLX9YBOCsBEgBL4rBONA
 OPENAI_BASE_URL=https://hubai.loe.gg/v1
 OPENAI_MODEL=gpt-4o-mini
 
-CV_AI_MODEL=your_ai_model
+CV_AI_MODEL=alibaba/tongyi-deepresearch-30b-a3b:free
 OPENROUTER_API_KEY=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

services/scraper-google/.env.example

@@ -0,0 +1,18 @@
+# Database Configuration
+DB_HOST=postgres
+DB_PORT=5432
+DB_NAME=jobs_scraper
+DB_USER=postgres
+DB_PASSWORD=postgres
+
+# OpenRouter Configuration (for job description extraction)
+OPENROUTER_API_KEY=your_openrouter_api_key_here
+CV_AI_MODEL=google/gemini-flash-1.5
+
+# Optional: OpenAI Configuration (alternative to OpenRouter)
+# OPENAI_API_KEY=your_openai_api_key_here
+# OPENAI_BASE_URL=https://api.openai.com/v1
+
+# Chrome/Chromedp Configuration (usually defaults are fine)
+CHROMEDP_DISABLE_GPU=true
+CHROMEDP_NO_SANDBOX=true

services/scraper-google/Dockerfile

@@ -0,0 +1,40 @@
+# Build stage
+FROM golang:1.24-alpine AS builder
+
+# Install build dependencies
+RUN apk add --no-cache git
+
+WORKDIR /build
+
+# Copy go mod files
+COPY go.mod go.sum ./
+RUN go mod download
+
+# Copy the entire monorepo
+COPY . .
+
+# Build the scraper-google service
+WORKDIR /build/services/scraper-google
+RUN CGO_ENABLED=0 GOOS=linux go build -o scraper-google .
+
+# Runtime stage
+FROM chromedp/headless-shell:latest
+
+# Install ca-certificates for HTTPS requests
+RUN apt-get update && apt-get install -y \
+    ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Copy the binary from builder
+COPY --from=builder /build/services/scraper-google/scraper-google .
+
+# Copy environment files if they exist (optional)
+COPY --from=builder /build/services/scraper-google/.env* ./
+
+# Expose any ports if needed (not required for this service)
+# EXPOSE 8080
+
+# Run the scraper
+CMD ["./scraper-google"]

services/scraper-google/Makefile

@@ -0,0 +1,67 @@
+.PHONY: help build up down logs restart clean rebuild
+
+help: ## Show this help message
+	@echo 'Usage: make [target]'
+	@echo ''
+	@echo 'Available targets:'
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "  %-15s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+
+build: ## Build the Docker images
+	docker-compose build
+
+up: ## Start all services
+	docker-compose up -d
+
+down: ## Stop all services
+	docker-compose down
+
+logs: ## View logs (use 'make logs service=scraper-google' for specific service)
+	@if [ -z "$(service)" ]; then \
+		docker-compose logs -f; \
+	else \
+		docker-compose logs -f $(service); \
+	fi
+
+restart: ## Restart all services
+	docker-compose restart
+
+restart-scraper: ## Restart just the scraper service
+	docker-compose restart scraper-google
+
+clean: ## Stop services and remove volumes (WARNING: deletes data)
+	docker-compose down -v
+
+rebuild: ## Rebuild and restart services
+	docker-compose down
+	docker-compose build --no-cache
+	docker-compose up -d
+
+ps: ## Show running containers
+	docker-compose ps
+
+stats: ## Show resource usage
+	docker stats
+
+exec-db: ## Connect to PostgreSQL database
+	docker-compose exec postgres psql -U postgres -d jobs_scraper
+
+backup-db: ## Backup the database
+	@mkdir -p backups
+	docker-compose exec postgres pg_dump -U postgres jobs_scraper > backups/backup_$(shell date +%Y%m%d_%H%M%S).sql
+	@echo "Database backed up to backups/backup_$(shell date +%Y%m%d_%H%M%S).sql"
+
+restore-db: ## Restore database from backup (use 'make restore-db file=backup.sql')
+	@if [ -z "$(file)" ]; then \
+		echo "Error: Please specify backup file. Usage: make restore-db file=backup.sql"; \
+		exit 1; \
+	fi
+	docker-compose exec -T postgres psql -U postgres jobs_scraper < $(file)
+
+shell-scraper: ## Open shell in scraper container
+	docker-compose exec scraper-google sh
+
+dev: ## Run in development mode with live logs
+	docker-compose up --build
+
+test: ## Run tests (if you have any)
+	docker-compose exec scraper-google go test ./...

services/scraper-google/docker-compose.yml

@@ -0,0 +1,49 @@
+version: "3.8"
+
+services:
+  scraper-google:
+    build:
+      context: ../../
+      dockerfile: services/scraper-google/Dockerfile
+    container_name: jobs-scraper-google
+    environment:
+      DB_HOST: postgres
+      DB_PORT: 5432
+      DB_NAME: jobs_scraper
+      DB_USER: scraper_user
+      DB_PASSWORD: Cocowawa_12345
+
+      # OpenRouter Configuration
+      OPENROUTER_API_KEY: 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
+      CV_AI_MODEL: alibaba/tongyi-deepresearch-30b-a3b:free
+
+      # Optional: OpenAI Configuration
+      OPENAI_API_KEY: sk-GtdLX9YBOCsBEgBL4rBONA
+      OPENAI_BASE_URL: https://hubai.loe.gg/v1
+      OPEN_AI_MODEL: gpt-4o-mini
+
+      # Chrome/Chromedp Configuration
+      CHROMEDP_DISABLE_GPU: true
+      CHROMEDP_NO_SANDBOX: true
+    volumes:
+      - ./.env:/app/.env:ro
+    restart: unless-stopped
+    networks:
+      - scraper-network
+    # Add resource limits to prevent runaway chrome instances
+    deploy:
+      resources:
+        limits:
+          memory: 2G
+          cpus: "2"
+        reservations:
+          memory: 512M
+          cpus: "0.5"
+
+volumes:
+  postgres_data:
+    driver: local
+
+networks:
+  scraper-network:
+    driver: bridge

services/scraper-google/main.go

@@ -53,7 +53,6 @@ func main() {
 		log.Fatalf("Failed to start scheduler: %v", err)
 	}
 
-	// Process and save jobs immediately as they're scraped
 	go func() {
 		for googleLink := range googleLinkStream {
 			htmlRaw, url, err := utils.GetHTMLRaw(ctx, googleLink)