jobs-monorepo/services/scraper-glassdoor/index.ts

import puppeteer, { Browser, Page } from "puppeteer-core";
import { ChildProcess } from "child_process";
import * as dotenv from "dotenv";
import {
  ChromeVersionResponse,
  launchChromeWithDebugging,
  randomDelay,
  waitForChromeReady,
} from "./utils/browser";
import {
  RabbitMQClient,
  SearchQuery,
  createRabbitMQClient,
} from "./utils/rabbitmq";
import { BrowserService } from "./services/browserService";
import { saveJobs } from "./queries/save-jobs";
import { saveJobDescriptions } from "./queries/save-job-description";

if (dotenv.config({ path: ".local.env" }).error) {
  console.log("No .local.env file found, trying .env");
  if (dotenv.config().error) {
    console.log("No .env file found, using system environment variables");
  }
}

let globalBrowser: Browser | null = null;
let chrome: ChildProcess;

async function cleanup() {
  console.log("Starting cleanup process...");

  if (rabbitMQClient) {
    console.log("Closing RabbitMQ connection");
    await rabbitMQClient.close();
    rabbitMQClient = null;
  }

  if (globalBrowser) {
    console.log("Closing browser on app shutdown");
    await globalBrowser.close();
    globalBrowser = null;
  }

  if (chrome) {
    chrome.kill("SIGINT");
  }

  console.log("Cleanup completed");
}

async function initializeBrowser(): Promise<{ browser: Browser; page: Page }> {
  chrome = launchChromeWithDebugging();
  await waitForChromeReady();

  const response = await fetch("http://localhost:9222/json/version", {
    headers: {
      Origin: "",
    },
  });

  if (!response.ok) {
    throw new Error("Failed to connect to Chrome debugging port");
  }

  const chromeInstance = (await response.json()) as ChromeVersionResponse;
  const { webSocketDebuggerUrl } = chromeInstance;

  const browser = await puppeteer.connect({
    browserWSEndpoint: webSocketDebuggerUrl,
    defaultViewport: null,
  });

  globalBrowser = browser;
  console.log("Connected to browser successfully");

  let pages = await browser.pages();
  console.log(`Found ${pages.length} pages`);

  let existingcookies = await browser.cookies();
  await browser.deleteCookie(...existingcookies);

  let page: Page;
  if (pages.length === 0) {
    console.log("No pages found, creating a new page");
    page = await browser.newPage();
  } else {
    page = pages[0];
  }

  await page.setUserAgent(
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
  );

  return { browser, page };
}

async function scrapeGlassdoorJobs(searchQuery: SearchQuery): Promise<void> {
  try {
    const { page } = await initializeBrowser();
    const browserService = new BrowserService(page);

    const searchUrl = `https://www.glassdoor.com/Job/jobs.htm`;

    await page.goto(searchUrl, {
      waitUntil: ["networkidle2", "domcontentloaded"],
      timeout: 60000,
    });

    await randomDelay(1500, 2500);

    await browserService.searchJobs(searchQuery.keywords, searchQuery.location);
    const jobs = await browserService.extractJobListings(searchQuery.location);

    if (jobs.length > 0) {
      await saveJobs(jobs);
      await saveJobDescriptions(jobs);

      console.log(
        `saved ${jobs.length} jobs from Glassdoor for query:`,
        searchQuery
      );
    } else {
      console.log("No jobs found for this search query");
    }

    await randomDelay(1500, 2500);
  } catch (error) {
    console.error("Error scraping Glassdoor jobs:", error);
    throw error;
  }

  if (globalBrowser) {
    console.log("Closing browser on app shutdown");
    await globalBrowser.close();
    globalBrowser = null;
  }

  if (chrome) {
    chrome.kill("SIGINT");
  }
}

let rabbitMQClient: RabbitMQClient | null = null;

async function subscribeToGlassdoorQueue(): Promise<void> {
  try {
    rabbitMQClient = await createRabbitMQClient();

    await rabbitMQClient.subscribe(async (searchQuery: SearchQuery) => {
      await scrapeGlassdoorJobs(searchQuery);
    });
  } catch (error) {
    console.error("Failed to set up RabbitMQ subscription:", error);
    process.exit(1);
  }
}

async function main() {
  console.log("Starting Glassdoor scraper service...");
  await subscribeToGlassdoorQueue();
}

process.on("SIGINT", async () => {
  console.log("Received SIGINT signal");
  await cleanup();
  if (chrome) {
    chrome.kill("SIGINT");
  }
  process.exit(0);
});

process.on("SIGTERM", async () => {
  console.log("Received SIGTERM signal");
  await cleanup();
  if (chrome) {
    chrome.kill("SIGTERM");
  }
  process.exit(0);
});

main().catch((error) => {
  console.error("Fatal error:", error);
  process.exit(1);
});