docker for api

.gitlab-ci.yml

@@ -1,23 +1,6 @@
 stages:
   - deploy
 
-deploy_production:
+include:
-  stage: deploy
+  - local: 'services/scraper-google/.gitlab-ci.yml'
-  image: alpine:3.20
+  - local: 'services/api/.gitlab-ci.yml'
-  environment:
-    name: production
-    url: https://jobs-scraper.ai-assistant-bot.xyz
-  rules:
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
-      when: manual
-      allow_failure: false
-  before_script:
-    - apk add --no-cache openssh-client rsync
-    - mkdir -p ~/.ssh
-    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_rsa
-    - chmod 600 ~/.ssh/id_rsa
-    - ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts
-  script:
-    - ssh "$DEPLOY_USER@$DEPLOY_HOST" "mkdir -p /opt/apps/jobs-scraper"
-    - rsync -az --delete --exclude='.git' --exclude='.env' --exclude='.env.*' --exclude='.local.env' ./ "$DEPLOY_USER@$DEPLOY_HOST:/opt/apps/jobs-scraper/"
-    - ssh "$DEPLOY_USER@$DEPLOY_HOST" "cd /opt/apps/jobs-scraper/services/scraper-google && docker compose up -d --build"

services/api/.gitlab-ci.yml

@@ -0,0 +1,20 @@
+deploy_api:
+  stage: deploy
+  image: alpine:3.20
+  environment:
+    name: production
+    url: https://jobs-scraper.ai-assistant-bot.xyz
+  rules:
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+      when: manual
+      allow_failure: false
+  before_script:
+    - apk add --no-cache openssh-client rsync
+    - mkdir -p ~/.ssh
+    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_rsa
+    - chmod 600 ~/.ssh/id_rsa
+    - ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts
+  script:
+    - ssh "$DEPLOY_USER@$DEPLOY_HOST" "mkdir -p /opt/apps/jobs-scraper"
+    - rsync -az --delete --exclude='.git' --exclude='.env' --exclude='.env.*' --exclude='.local.env' ./ "$DEPLOY_USER@$DEPLOY_HOST:/opt/apps/jobs-scraper/"
+    - ssh "$DEPLOY_USER@$DEPLOY_HOST" "cd /opt/apps/jobs-scraper/services/api && docker compose up -d --build"

services/api/Dockerfile

@@ -0,0 +1,33 @@
+# Build stage
+FROM golang:1.24-alpine AS builder
+
+# Install build dependencies
+RUN apk add --no-cache git
+
+WORKDIR /build
+
+# Copy go mod files
+COPY go.mod go.sum ./
+RUN go mod download
+
+# Copy the entire monorepo
+COPY . .
+
+# Build the API service
+WORKDIR /build/services/api
+RUN CGO_ENABLED=0 GOOS=linux go build -o api ./cmd/server
+
+# Runtime stage
+FROM alpine:latest
+
+RUN apk add --no-cache ca-certificates
+
+WORKDIR /app
+
+# Copy the binary from builder
+COPY --from=builder /build/services/api/api .
+
+EXPOSE 8080
+
+# Run the API
+ENTRYPOINT ["./api"]

services/api/docker-compose.yml

@@ -0,0 +1,45 @@
+version: "3.8"
+
+services:
+  api:
+    build:
+      context: ../../
+      dockerfile: services/api/Dockerfile
+    container_name: jobs-api
+    ports:
+      - "8080:8080"
+    environment:
+      DB_HOST: shared_postgres
+      DB_PORT: 5432
+      DB_NAME: jobs_scraper
+      DB_USER: scraper_user
+      DB_PASSWORD: Cocowawa_12345
+
+      # RabbitMQ Configuration
+      RABBITMQ_HOST: rabbitmq
+      RABBITMQ_PORT: 5672
+      RABBITMQ_USER: admin
+      RABBITMQ_PASSWORD: 123Kari123!
+    networks:
+      - proxy
+    restart: unless-stopped
+    labels:
+      - traefik.enable=true
+      - traefik.docker.network=proxy
+      - traefik.http.routers.api-jobs.rule=Host(`api-jobs.ai-assistant-bot.xyz`)
+      - traefik.http.routers.api-jobs.entrypoints=web,websecure
+      - traefik.http.routers.api-jobs.tls.certresolver=le
+      - traefik.http.services.api-jobs.loadbalancer.server.port=8080
+    depends_on:
+      - rabbitmq
+    # deploy:
+    #   resources:
+    #     limits:
+    #       memory: 512M
+    #       cpus: "1"
+    #     reservations:
+    #       memory: 128M
+    #       cpus: "0.25"
+networks:
+  proxy:
+    external: true

services/scraper-google/.gitlab-ci.yml

@@ -0,0 +1,20 @@
+deploy_scraper_google:
+  stage: deploy
+  image: alpine:3.20
+  environment:
+    name: production
+    url: https://jobs-scraper.ai-assistant-bot.xyz
+  rules:
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+      when: manual
+      allow_failure: false
+  before_script:
+    - apk add --no-cache openssh-client rsync
+    - mkdir -p ~/.ssh
+    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_rsa
+    - chmod 600 ~/.ssh/id_rsa
+    - ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts
+  script:
+    - ssh "$DEPLOY_USER@$DEPLOY_HOST" "mkdir -p /opt/apps/jobs-scraper"
+    - rsync -az --delete --exclude='.git' --exclude='.env' --exclude='.env.*' --exclude='.local.env' ./ "$DEPLOY_USER@$DEPLOY_HOST:/opt/apps/jobs-scraper/"
+    - ssh "$DEPLOY_USER@$DEPLOY_HOST" "cd /opt/apps/jobs-scraper/services/scraper-google && docker compose up -d --build"

services/scraper-google/main.go

@@ -49,8 +49,8 @@ func main() {
 
 	sched := sked.New(ctx)
 	googleLinkStream := make(chan domain.GoogleLink, 100)
-	// pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}
+	pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}
-	pages := []int{100, 110, 120, 130}
+	// pages := []int{100, 110, 120, 130}
 
 	log.Println("Starting scraper with scheduler...")
 
@@ -174,7 +174,7 @@ func launchChrome() (string, *exec.Cmd, error) {
 		"--safebrowsing-disable-auto-update",
 		"--user-data-dir=/tmp/chrome-debug-profile",
 		// "--incognito",
-		// "--headless",
+		"--headless",
 	)
 
 	if err := cmd.Start(); err != nil {