deploy
This commit is contained in:
Elshimy Ziad Magdy Taha
parent
1dd59249c5
commit
2589c81797
8 changed files with 245 additions and 2 deletions
23
.gitlab-ci.yml
Normal file
23
.gitlab-ci.yml
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
stages:
|
||||
- deploy
|
||||
|
||||
deploy_production:
|
||||
stage: deploy
|
||||
image: alpine:3.20
|
||||
environment:
|
||||
name: production
|
||||
url: https://jobs-scraper.ai-assistant-bot.xyz
|
||||
rules:
|
||||
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
|
||||
when: manual
|
||||
allow_failure: false
|
||||
before_script:
|
||||
- apk add --no-cache openssh-client rsync
|
||||
- mkdir -p ~/.ssh
|
||||
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_rsa
|
||||
- chmod 600 ~/.ssh/id_rsa
|
||||
- ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts
|
||||
script:
|
||||
- ssh "$DEPLOY_USER@$DEPLOY_HOST" "mkdir -p /opt/apps/jobs-scraper"
|
||||
- rsync -az --delete --exclude='.git' --exclude='.env' --exclude='.env.*' --exclude='.local.env' ./ "$DEPLOY_USER@$DEPLOY_HOST:/opt/apps/jobs-scraper/"
|
||||
- ssh "$DEPLOY_USER@$DEPLOY_HOST" "cd /opt/apps/jobs-scraper/services/scraper-google && docker compose up -d --build"
|
||||
47
services/scraper-google/.dockerignore
Normal file
47
services/scraper-google/.dockerignore
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
# Git
|
||||
.git
|
||||
.gitignore
|
||||
|
||||
# Environment files (will be mounted or set via docker-compose)
|
||||
.local.env
|
||||
*.env
|
||||
|
||||
# Documentation
|
||||
*.md
|
||||
DOCKER_README.md
|
||||
RECOMMENDED_MODELS.md
|
||||
|
||||
# Development files
|
||||
.vscode
|
||||
.idea
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# Build artifacts
|
||||
*.exe
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
scraper-google
|
||||
|
||||
# Test files
|
||||
*_test.go
|
||||
testdata/
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
logs/
|
||||
|
||||
# Temporary files
|
||||
tmp/
|
||||
temp/
|
||||
|
||||
# OS files
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Docker files (not needed in image)
|
||||
Dockerfile
|
||||
docker-compose.yml
|
||||
.dockerignore
|
||||
|
|
@ -12,5 +12,5 @@ OPENAI_API_KEY=sk-GtdLX9YBOCsBEgBL4rBONA
|
|||
OPENAI_BASE_URL=https://hubai.loe.gg/v1
|
||||
OPENAI_MODEL=gpt-4o-mini
|
||||
|
||||
CV_AI_MODEL=your_ai_model
|
||||
CV_AI_MODEL=alibaba/tongyi-deepresearch-30b-a3b:free
|
||||
OPENROUTER_API_KEY=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
|
||||
18
services/scraper-google/.env.example
Normal file
18
services/scraper-google/.env.example
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
# Database Configuration
|
||||
DB_HOST=postgres
|
||||
DB_PORT=5432
|
||||
DB_NAME=jobs_scraper
|
||||
DB_USER=postgres
|
||||
DB_PASSWORD=postgres
|
||||
|
||||
# OpenRouter Configuration (for job description extraction)
|
||||
OPENROUTER_API_KEY=your_openrouter_api_key_here
|
||||
CV_AI_MODEL=google/gemini-flash-1.5
|
||||
|
||||
# Optional: OpenAI Configuration (alternative to OpenRouter)
|
||||
# OPENAI_API_KEY=your_openai_api_key_here
|
||||
# OPENAI_BASE_URL=https://api.openai.com/v1
|
||||
|
||||
# Chrome/Chromedp Configuration (usually defaults are fine)
|
||||
CHROMEDP_DISABLE_GPU=true
|
||||
CHROMEDP_NO_SANDBOX=true
|
||||
40
services/scraper-google/Dockerfile
Normal file
40
services/scraper-google/Dockerfile
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
# Build stage
|
||||
FROM golang:1.24-alpine AS builder
|
||||
|
||||
# Install build dependencies
|
||||
RUN apk add --no-cache git
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
# Copy go mod files
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
# Copy the entire monorepo
|
||||
COPY . .
|
||||
|
||||
# Build the scraper-google service
|
||||
WORKDIR /build/services/scraper-google
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -o scraper-google .
|
||||
|
||||
# Runtime stage
|
||||
FROM chromedp/headless-shell:latest
|
||||
|
||||
# Install ca-certificates for HTTPS requests
|
||||
RUN apt-get update && apt-get install -y \
|
||||
ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy the binary from builder
|
||||
COPY --from=builder /build/services/scraper-google/scraper-google .
|
||||
|
||||
# Copy environment files if they exist (optional)
|
||||
COPY --from=builder /build/services/scraper-google/.env* ./
|
||||
|
||||
# Expose any ports if needed (not required for this service)
|
||||
# EXPOSE 8080
|
||||
|
||||
# Run the scraper
|
||||
CMD ["./scraper-google"]
|
||||
67
services/scraper-google/Makefile
Normal file
67
services/scraper-google/Makefile
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
.PHONY: help build up down logs restart clean rebuild
|
||||
|
||||
help: ## Show this help message
|
||||
@echo 'Usage: make [target]'
|
||||
@echo ''
|
||||
@echo 'Available targets:'
|
||||
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf " %-15s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
|
||||
|
||||
build: ## Build the Docker images
|
||||
docker-compose build
|
||||
|
||||
up: ## Start all services
|
||||
docker-compose up -d
|
||||
|
||||
down: ## Stop all services
|
||||
docker-compose down
|
||||
|
||||
logs: ## View logs (use 'make logs service=scraper-google' for specific service)
|
||||
@if [ -z "$(service)" ]; then \
|
||||
docker-compose logs -f; \
|
||||
else \
|
||||
docker-compose logs -f $(service); \
|
||||
fi
|
||||
|
||||
restart: ## Restart all services
|
||||
docker-compose restart
|
||||
|
||||
restart-scraper: ## Restart just the scraper service
|
||||
docker-compose restart scraper-google
|
||||
|
||||
clean: ## Stop services and remove volumes (WARNING: deletes data)
|
||||
docker-compose down -v
|
||||
|
||||
rebuild: ## Rebuild and restart services
|
||||
docker-compose down
|
||||
docker-compose build --no-cache
|
||||
docker-compose up -d
|
||||
|
||||
ps: ## Show running containers
|
||||
docker-compose ps
|
||||
|
||||
stats: ## Show resource usage
|
||||
docker stats
|
||||
|
||||
exec-db: ## Connect to PostgreSQL database
|
||||
docker-compose exec postgres psql -U postgres -d jobs_scraper
|
||||
|
||||
backup-db: ## Backup the database
|
||||
@mkdir -p backups
|
||||
docker-compose exec postgres pg_dump -U postgres jobs_scraper > backups/backup_$(shell date +%Y%m%d_%H%M%S).sql
|
||||
@echo "Database backed up to backups/backup_$(shell date +%Y%m%d_%H%M%S).sql"
|
||||
|
||||
restore-db: ## Restore database from backup (use 'make restore-db file=backup.sql')
|
||||
@if [ -z "$(file)" ]; then \
|
||||
echo "Error: Please specify backup file. Usage: make restore-db file=backup.sql"; \
|
||||
exit 1; \
|
||||
fi
|
||||
docker-compose exec -T postgres psql -U postgres jobs_scraper < $(file)
|
||||
|
||||
shell-scraper: ## Open shell in scraper container
|
||||
docker-compose exec scraper-google sh
|
||||
|
||||
dev: ## Run in development mode with live logs
|
||||
docker-compose up --build
|
||||
|
||||
test: ## Run tests (if you have any)
|
||||
docker-compose exec scraper-google go test ./...
|
||||
49
services/scraper-google/docker-compose.yml
Normal file
49
services/scraper-google/docker-compose.yml
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
version: "3.8"
|
||||
|
||||
services:
|
||||
scraper-google:
|
||||
build:
|
||||
context: ../../
|
||||
dockerfile: services/scraper-google/Dockerfile
|
||||
container_name: jobs-scraper-google
|
||||
environment:
|
||||
DB_HOST: postgres
|
||||
DB_PORT: 5432
|
||||
DB_NAME: jobs_scraper
|
||||
DB_USER: scraper_user
|
||||
DB_PASSWORD: Cocowawa_12345
|
||||
|
||||
# OpenRouter Configuration
|
||||
OPENROUTER_API_KEY: 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
|
||||
CV_AI_MODEL: alibaba/tongyi-deepresearch-30b-a3b:free
|
||||
|
||||
# Optional: OpenAI Configuration
|
||||
OPENAI_API_KEY: sk-GtdLX9YBOCsBEgBL4rBONA
|
||||
OPENAI_BASE_URL: https://hubai.loe.gg/v1
|
||||
OPEN_AI_MODEL: gpt-4o-mini
|
||||
|
||||
# Chrome/Chromedp Configuration
|
||||
CHROMEDP_DISABLE_GPU: true
|
||||
CHROMEDP_NO_SANDBOX: true
|
||||
volumes:
|
||||
- ./.env:/app/.env:ro
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- scraper-network
|
||||
# Add resource limits to prevent runaway chrome instances
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
memory: 2G
|
||||
cpus: "2"
|
||||
reservations:
|
||||
memory: 512M
|
||||
cpus: "0.5"
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
driver: local
|
||||
|
||||
networks:
|
||||
scraper-network:
|
||||
driver: bridge
|
||||
|
|
@ -53,7 +53,6 @@ func main() {
|
|||
log.Fatalf("Failed to start scheduler: %v", err)
|
||||
}
|
||||
|
||||
// Process and save jobs immediately as they're scraped
|
||||
go func() {
|
||||
for googleLink := range googleLinkStream {
|
||||
htmlRaw, url, err := utils.GetHTMLRaw(ctx, googleLink)
|
||||
|
|
|
|||
Loading…
Reference in a new issue