docker for api
This commit is contained in:
Elshimy Ziad Magdy Taha
parent
a8c253bdd5
commit
3187757776
6 changed files with 124 additions and 23 deletions
|
|
@ -1,23 +1,6 @@
|
|||
stages:
|
||||
- deploy
|
||||
|
||||
deploy_production:
|
||||
stage: deploy
|
||||
image: alpine:3.20
|
||||
environment:
|
||||
name: production
|
||||
url: https://jobs-scraper.ai-assistant-bot.xyz
|
||||
rules:
|
||||
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
|
||||
when: manual
|
||||
allow_failure: false
|
||||
before_script:
|
||||
- apk add --no-cache openssh-client rsync
|
||||
- mkdir -p ~/.ssh
|
||||
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_rsa
|
||||
- chmod 600 ~/.ssh/id_rsa
|
||||
- ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts
|
||||
script:
|
||||
- ssh "$DEPLOY_USER@$DEPLOY_HOST" "mkdir -p /opt/apps/jobs-scraper"
|
||||
- rsync -az --delete --exclude='.git' --exclude='.env' --exclude='.env.*' --exclude='.local.env' ./ "$DEPLOY_USER@$DEPLOY_HOST:/opt/apps/jobs-scraper/"
|
||||
- ssh "$DEPLOY_USER@$DEPLOY_HOST" "cd /opt/apps/jobs-scraper/services/scraper-google && docker compose up -d --build"
|
||||
include:
|
||||
- local: 'services/scraper-google/.gitlab-ci.yml'
|
||||
- local: 'services/api/.gitlab-ci.yml'
|
||||
|
|
|
|||
20
services/api/.gitlab-ci.yml
Normal file
20
services/api/.gitlab-ci.yml
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
deploy_api:
|
||||
stage: deploy
|
||||
image: alpine:3.20
|
||||
environment:
|
||||
name: production
|
||||
url: https://jobs-scraper.ai-assistant-bot.xyz
|
||||
rules:
|
||||
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
|
||||
when: manual
|
||||
allow_failure: false
|
||||
before_script:
|
||||
- apk add --no-cache openssh-client rsync
|
||||
- mkdir -p ~/.ssh
|
||||
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_rsa
|
||||
- chmod 600 ~/.ssh/id_rsa
|
||||
- ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts
|
||||
script:
|
||||
- ssh "$DEPLOY_USER@$DEPLOY_HOST" "mkdir -p /opt/apps/jobs-scraper"
|
||||
- rsync -az --delete --exclude='.git' --exclude='.env' --exclude='.env.*' --exclude='.local.env' ./ "$DEPLOY_USER@$DEPLOY_HOST:/opt/apps/jobs-scraper/"
|
||||
- ssh "$DEPLOY_USER@$DEPLOY_HOST" "cd /opt/apps/jobs-scraper/services/api && docker compose up -d --build"
|
||||
33
services/api/Dockerfile
Normal file
33
services/api/Dockerfile
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
# Build stage
|
||||
FROM golang:1.24-alpine AS builder
|
||||
|
||||
# Install build dependencies
|
||||
RUN apk add --no-cache git
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
# Copy go mod files
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
# Copy the entire monorepo
|
||||
COPY . .
|
||||
|
||||
# Build the API service
|
||||
WORKDIR /build/services/api
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -o api ./cmd/server
|
||||
|
||||
# Runtime stage
|
||||
FROM alpine:latest
|
||||
|
||||
RUN apk add --no-cache ca-certificates
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy the binary from builder
|
||||
COPY --from=builder /build/services/api/api .
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
# Run the API
|
||||
ENTRYPOINT ["./api"]
|
||||
45
services/api/docker-compose.yml
Normal file
45
services/api/docker-compose.yml
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
version: "3.8"
|
||||
|
||||
services:
|
||||
api:
|
||||
build:
|
||||
context: ../../
|
||||
dockerfile: services/api/Dockerfile
|
||||
container_name: jobs-api
|
||||
ports:
|
||||
- "8080:8080"
|
||||
environment:
|
||||
DB_HOST: shared_postgres
|
||||
DB_PORT: 5432
|
||||
DB_NAME: jobs_scraper
|
||||
DB_USER: scraper_user
|
||||
DB_PASSWORD: Cocowawa_12345
|
||||
|
||||
# RabbitMQ Configuration
|
||||
RABBITMQ_HOST: rabbitmq
|
||||
RABBITMQ_PORT: 5672
|
||||
RABBITMQ_USER: admin
|
||||
RABBITMQ_PASSWORD: 123Kari123!
|
||||
networks:
|
||||
- proxy
|
||||
restart: unless-stopped
|
||||
labels:
|
||||
- traefik.enable=true
|
||||
- traefik.docker.network=proxy
|
||||
- traefik.http.routers.api-jobs.rule=Host(`api-jobs.ai-assistant-bot.xyz`)
|
||||
- traefik.http.routers.api-jobs.entrypoints=web,websecure
|
||||
- traefik.http.routers.api-jobs.tls.certresolver=le
|
||||
- traefik.http.services.api-jobs.loadbalancer.server.port=8080
|
||||
depends_on:
|
||||
- rabbitmq
|
||||
# deploy:
|
||||
# resources:
|
||||
# limits:
|
||||
# memory: 512M
|
||||
# cpus: "1"
|
||||
# reservations:
|
||||
# memory: 128M
|
||||
# cpus: "0.25"
|
||||
networks:
|
||||
proxy:
|
||||
external: true
|
||||
20
services/scraper-google/.gitlab-ci.yml
Normal file
20
services/scraper-google/.gitlab-ci.yml
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
deploy_scraper_google:
|
||||
stage: deploy
|
||||
image: alpine:3.20
|
||||
environment:
|
||||
name: production
|
||||
url: https://jobs-scraper.ai-assistant-bot.xyz
|
||||
rules:
|
||||
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
|
||||
when: manual
|
||||
allow_failure: false
|
||||
before_script:
|
||||
- apk add --no-cache openssh-client rsync
|
||||
- mkdir -p ~/.ssh
|
||||
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_rsa
|
||||
- chmod 600 ~/.ssh/id_rsa
|
||||
- ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts
|
||||
script:
|
||||
- ssh "$DEPLOY_USER@$DEPLOY_HOST" "mkdir -p /opt/apps/jobs-scraper"
|
||||
- rsync -az --delete --exclude='.git' --exclude='.env' --exclude='.env.*' --exclude='.local.env' ./ "$DEPLOY_USER@$DEPLOY_HOST:/opt/apps/jobs-scraper/"
|
||||
- ssh "$DEPLOY_USER@$DEPLOY_HOST" "cd /opt/apps/jobs-scraper/services/scraper-google && docker compose up -d --build"
|
||||
|
|
@ -49,8 +49,8 @@ func main() {
|
|||
|
||||
sched := sked.New(ctx)
|
||||
googleLinkStream := make(chan domain.GoogleLink, 100)
|
||||
// pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}
|
||||
pages := []int{100, 110, 120, 130}
|
||||
pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}
|
||||
// pages := []int{100, 110, 120, 130}
|
||||
|
||||
log.Println("Starting scraper with scheduler...")
|
||||
|
||||
|
|
@ -174,7 +174,7 @@ func launchChrome() (string, *exec.Cmd, error) {
|
|||
"--safebrowsing-disable-auto-update",
|
||||
"--user-data-dir=/tmp/chrome-debug-profile",
|
||||
// "--incognito",
|
||||
// "--headless",
|
||||
"--headless",
|
||||
)
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
|
|
|
|||
Loading…
Reference in a new issue