diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 59f05d7..5b94cfc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,23 +1,6 @@ stages: - deploy -deploy_production: - stage: deploy - image: alpine:3.20 - environment: - name: production - url: https://jobs-scraper.ai-assistant-bot.xyz - rules: - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' - when: manual - allow_failure: false - before_script: - - apk add --no-cache openssh-client rsync - - mkdir -p ~/.ssh - - echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_rsa - - chmod 600 ~/.ssh/id_rsa - - ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts - script: - - ssh "$DEPLOY_USER@$DEPLOY_HOST" "mkdir -p /opt/apps/jobs-scraper" - - rsync -az --delete --exclude='.git' --exclude='.env' --exclude='.env.*' --exclude='.local.env' ./ "$DEPLOY_USER@$DEPLOY_HOST:/opt/apps/jobs-scraper/" - - ssh "$DEPLOY_USER@$DEPLOY_HOST" "cd /opt/apps/jobs-scraper/services/scraper-google && docker compose up -d --build" +include: + - local: 'services/scraper-google/.gitlab-ci.yml' + - local: 'services/api/.gitlab-ci.yml' diff --git a/services/api/.gitlab-ci.yml b/services/api/.gitlab-ci.yml new file mode 100644 index 0000000..5c08830 --- /dev/null +++ b/services/api/.gitlab-ci.yml @@ -0,0 +1,20 @@ +deploy_api: + stage: deploy + image: alpine:3.20 + environment: + name: production + url: https://jobs-scraper.ai-assistant-bot.xyz + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + when: manual + allow_failure: false + before_script: + - apk add --no-cache openssh-client rsync + - mkdir -p ~/.ssh + - echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_rsa + - chmod 600 ~/.ssh/id_rsa + - ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts + script: + - ssh "$DEPLOY_USER@$DEPLOY_HOST" "mkdir -p /opt/apps/jobs-scraper" + - rsync -az --delete --exclude='.git' --exclude='.env' --exclude='.env.*' --exclude='.local.env' ./ "$DEPLOY_USER@$DEPLOY_HOST:/opt/apps/jobs-scraper/" + - ssh "$DEPLOY_USER@$DEPLOY_HOST" "cd /opt/apps/jobs-scraper/services/api && docker compose up -d --build" diff --git a/services/api/Dockerfile b/services/api/Dockerfile new file mode 100644 index 0000000..56c77da --- /dev/null +++ b/services/api/Dockerfile @@ -0,0 +1,33 @@ +# Build stage +FROM golang:1.24-alpine AS builder + +# Install build dependencies +RUN apk add --no-cache git + +WORKDIR /build + +# Copy go mod files +COPY go.mod go.sum ./ +RUN go mod download + +# Copy the entire monorepo +COPY . . + +# Build the API service +WORKDIR /build/services/api +RUN CGO_ENABLED=0 GOOS=linux go build -o api ./cmd/server + +# Runtime stage +FROM alpine:latest + +RUN apk add --no-cache ca-certificates + +WORKDIR /app + +# Copy the binary from builder +COPY --from=builder /build/services/api/api . + +EXPOSE 8080 + +# Run the API +ENTRYPOINT ["./api"] diff --git a/services/api/docker-compose.yml b/services/api/docker-compose.yml new file mode 100644 index 0000000..2ab899c --- /dev/null +++ b/services/api/docker-compose.yml @@ -0,0 +1,45 @@ +version: "3.8" + +services: + api: + build: + context: ../../ + dockerfile: services/api/Dockerfile + container_name: jobs-api + ports: + - "8080:8080" + environment: + DB_HOST: shared_postgres + DB_PORT: 5432 + DB_NAME: jobs_scraper + DB_USER: scraper_user + DB_PASSWORD: Cocowawa_12345 + + # RabbitMQ Configuration + RABBITMQ_HOST: rabbitmq + RABBITMQ_PORT: 5672 + RABBITMQ_USER: admin + RABBITMQ_PASSWORD: 123Kari123! + networks: + - proxy + restart: unless-stopped + labels: + - traefik.enable=true + - traefik.docker.network=proxy + - traefik.http.routers.api-jobs.rule=Host(`api-jobs.ai-assistant-bot.xyz`) + - traefik.http.routers.api-jobs.entrypoints=web,websecure + - traefik.http.routers.api-jobs.tls.certresolver=le + - traefik.http.services.api-jobs.loadbalancer.server.port=8080 + depends_on: + - rabbitmq + # deploy: + # resources: + # limits: + # memory: 512M + # cpus: "1" + # reservations: + # memory: 128M + # cpus: "0.25" +networks: + proxy: + external: true diff --git a/services/scraper-google/.gitlab-ci.yml b/services/scraper-google/.gitlab-ci.yml new file mode 100644 index 0000000..b956508 --- /dev/null +++ b/services/scraper-google/.gitlab-ci.yml @@ -0,0 +1,20 @@ +deploy_scraper_google: + stage: deploy + image: alpine:3.20 + environment: + name: production + url: https://jobs-scraper.ai-assistant-bot.xyz + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + when: manual + allow_failure: false + before_script: + - apk add --no-cache openssh-client rsync + - mkdir -p ~/.ssh + - echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_rsa + - chmod 600 ~/.ssh/id_rsa + - ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts + script: + - ssh "$DEPLOY_USER@$DEPLOY_HOST" "mkdir -p /opt/apps/jobs-scraper" + - rsync -az --delete --exclude='.git' --exclude='.env' --exclude='.env.*' --exclude='.local.env' ./ "$DEPLOY_USER@$DEPLOY_HOST:/opt/apps/jobs-scraper/" + - ssh "$DEPLOY_USER@$DEPLOY_HOST" "cd /opt/apps/jobs-scraper/services/scraper-google && docker compose up -d --build" diff --git a/services/scraper-google/main.go b/services/scraper-google/main.go index adb30f9..107f50b 100644 --- a/services/scraper-google/main.go +++ b/services/scraper-google/main.go @@ -49,8 +49,8 @@ func main() { sched := sked.New(ctx) googleLinkStream := make(chan domain.GoogleLink, 100) - // pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} - pages := []int{100, 110, 120, 130} + pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} + // pages := []int{100, 110, 120, 130} log.Println("Starting scraper with scheduler...") @@ -174,7 +174,7 @@ func launchChrome() (string, *exec.Cmd, error) { "--safebrowsing-disable-auto-update", "--user-data-dir=/tmp/chrome-debug-profile", // "--incognito", - // "--headless", + "--headless", ) if err := cmd.Start(); err != nil {