jobs-monorepo/packages/glassdoor-scraper/index.ts

import puppeteer, { Browser, Page } from "puppeteer-core";
import { ChildProcess } from "child_process";
import * as dotenv from "dotenv";
import {
  ChromeVersionResponse,
  launchChromeWithDebugging,
  randomDelay,
  waitForChromeReady,
} from "./utils/browser";
import {
  RabbitMQClient,
  SearchQuery,
  createRabbitMQClient,
} from "./utils/rabbitmq";
import { BrowserService } from "./services/browserService";

dotenv.config();

let globalBrowser: Browser | null = null;
let chrome: ChildProcess;

async function cleanup() {
  console.log("Starting cleanup process...");

  if (rabbitMQClient) {
    console.log("Closing RabbitMQ connection");
    await rabbitMQClient.close();
    rabbitMQClient = null;
  }

  if (globalBrowser) {
    console.log("Closing browser on app shutdown");
    await globalBrowser.close();
    globalBrowser = null;
  }

  if (chrome) {
    chrome.kill("SIGINT");
  }

  console.log("Cleanup completed");
}

async function initializeBrowser(): Promise<{ browser: Browser; page: Page }> {
  chrome = launchChromeWithDebugging();
  await waitForChromeReady();

  const response = await fetch("http://localhost:9222/json/version", {
    headers: {
      Origin: "",
    },
  });

  if (!response.ok) {
    throw new Error("Failed to connect to Chrome debugging port");
  }

  const chromeInstance = (await response.json()) as ChromeVersionResponse;
  const { webSocketDebuggerUrl } = chromeInstance;

  const browser = await puppeteer.connect({
    browserWSEndpoint: webSocketDebuggerUrl,
    defaultViewport: null,
  });

  globalBrowser = browser;
  console.log("Connected to browser successfully");

  let pages = await browser.pages();
  console.log(`Found ${pages.length} pages`);

  let existingcookies = await browser.cookies();
  await browser.deleteCookie(...existingcookies);

  let page: Page;
  if (pages.length === 0) {
    console.log("No pages found, creating a new page");
    page = await browser.newPage();
  } else {
    page = pages[0];
  }

  await page.setUserAgent(
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
  );

  return { browser, page };
}

async function scrapeGlassdoorJobs(searchQuery: SearchQuery): Promise<void> {
  console.log(
    `Starting Glassdoor scrape for: ${searchQuery.keywords} in ${searchQuery.location}`
  );

  try {
    const { page } = await initializeBrowser();
    const browserService = new BrowserService(page);

    // Build Glassdoor search URL
    const searchUrl = buildGlassdoorSearchURL(searchQuery);
    console.log(`Navigating to: ${searchUrl}`);

    await page.goto(searchUrl, {
      waitUntil: ["networkidle2", "domcontentloaded"],
      timeout: 60000,
    });

    await randomDelay(1500, 2500);

    await browserService.searchJobs(searchQuery.keywords, searchQuery.location);
    const jobs = await browserService.extractJobListings();

    console.log(`\n=== SCRAPING RESULTS ===`);
    console.log(`Successfully extracted ${jobs.length} jobs from Glassdoor`);

    if (jobs.length > 0) {
      console.log(`\n=== JOB LISTINGS ===`);
      jobs.forEach((job, index) => {
        console.log(`\n${index + 1}. ${job.title}`);
        console.log(`   Company: ${job.company}`);
        console.log(`   Location: ${job.location}`);
        console.log(`   Age: ${job.age}`);
        console.log(`   Job ID: ${job.jobId}`);
        if (job.skills.length > 0) {
          console.log(`   Skills: ${job.skills.join(", ")}`);
        }
        if (job.jobLink) {
          console.log(`   Job Link: ${job.jobLink}`);
        }
        if (job.description) {
          console.log(
            `   Description: ${job.description.substring(0, 200)}...`
          );
        }
      });

      // Here you can process the jobs further:
      // - Save to database
      // - Send to another queue
      // - Perform additional analysis
      console.log(`\n=== END RESULTS ===`);
    } else {
      console.log("No jobs found for this search query");
    }

    await randomDelay(1500, 2500);
  } catch (error) {
    console.error("Error scraping Glassdoor jobs:", error);
    throw error;
  }
}

function getGlassdoorDomain(location: string): string {
  const locationLower = location.toLowerCase();

  // Country-specific Glassdoor domains
  const countryDomains: { [key: string]: string } = {
    // Asia Pacific
    japan: "https://www.glassdoor.com/Job", // Glassdoor Japan uses .com
    singapore: "https://www.glassdoor.sg/Job",
    australia: "https://www.glassdoor.com.au/Job",
    "hong kong": "https://www.glassdoor.com/Job", // Uses .com with HK location
    india: "https://www.glassdoor.co.in/Job",

    // Europe
    "united kingdom": "https://www.glassdoor.co.uk/Job",
    uk: "https://www.glassdoor.co.uk/Job",
    germany: "https://www.glassdoor.de/Job",
    france: "https://www.glassdoor.fr/Job",
    netherlands: "https://www.glassdoor.nl/Job",
    belgium: "https://www.glassdoor.be/Job",
    austria: "https://www.glassdoor.at/Job",
    switzerland: "https://www.glassdoor.ch/Job",
    ireland: "https://www.glassdoor.ie/Job",
    spain: "https://www.glassdoor.es/Job",
    italy: "https://www.glassdoor.it/Job",

    // North America
    canada: "https://www.glassdoor.ca/Job",
    mexico: "https://www.glassdoor.com.mx/Job",

    // South America
    brazil: "https://www.glassdoor.com.br/Job",
    argentina: "https://www.glassdoor.com.ar/Job",
  };

  // Check for country matches
  for (const [country, domain] of Object.entries(countryDomains)) {
    if (locationLower.includes(country)) {
      return domain;
    }
  }

  // Check for city-specific mappings
  const cityMappings: { [key: string]: string } = {
    tokyo: "https://www.glassdoor.com/Job",
    osaka: "https://www.glassdoor.com/Job",
    kyoto: "https://www.glassdoor.com/Job",
    london: "https://www.glassdoor.co.uk/Job",
    manchester: "https://www.glassdoor.co.uk/Job",
    berlin: "https://www.glassdoor.de/Job",
    munich: "https://www.glassdoor.de/Job",
    paris: "https://www.glassdoor.fr/Job",
    amsterdam: "https://www.glassdoor.nl/Job",
    toronto: "https://www.glassdoor.ca/Job",
    vancouver: "https://www.glassdoor.ca/Job",
    sydney: "https://www.glassdoor.com.au/Job",
    melbourne: "https://www.glassdoor.com.au/Job",
    singapore: "https://www.glassdoor.sg/Job",
    mumbai: "https://www.glassdoor.co.in/Job",
    bangalore: "https://www.glassdoor.co.in/Job",
    delhi: "https://www.glassdoor.co.in/Job",
  };

  for (const [city, domain] of Object.entries(cityMappings)) {
    if (locationLower.includes(city)) {
      return domain;
    }
  }

  // Default to US site
  return "https://www.glassdoor.com/Job";
}

function buildGlassdoorSearchURL(query: SearchQuery): string {
  const baseDomain = getGlassdoorDomain(query.location);
  const baseUrl = `${baseDomain}/jobs.htm`;

  console.log(
    `Selected Glassdoor domain for location "${query.location}": ${baseDomain}`
  );

  const params = new URLSearchParams();

  // if (query.Keywords) {
  //   params.set("sc.keyword", query.Keywords);
  // }
  // if (query.Location) {
  //   params.set("locT", "C");
  //   params.set("locId", query.Location);
  // }

  return `${baseUrl}?${params.toString()}`;
}

let rabbitMQClient: RabbitMQClient | null = null;

async function subscribeToGlassdoorQueue(): Promise<void> {
  try {
    rabbitMQClient = await createRabbitMQClient();

    await rabbitMQClient.subscribe(async (searchQuery: SearchQuery) => {
      await scrapeGlassdoorJobs(searchQuery);
    });
  } catch (error) {
    console.error("Failed to set up RabbitMQ subscription:", error);
    process.exit(1);
  }
}

async function main() {
  console.log("Starting Glassdoor scraper service...");
  await subscribeToGlassdoorQueue();
}

process.on("SIGINT", async () => {
  console.log("Received SIGINT signal");
  await cleanup();
  if (chrome) {
    chrome.kill("SIGINT");
  }
  process.exit(0);
});

process.on("SIGTERM", async () => {
  console.log("Received SIGTERM signal");
  await cleanup();
  if (chrome) {
    chrome.kill("SIGTERM");
  }
  process.exit(0);
});

main().catch((error) => {
  console.error("Fatal error:", error);
  process.exit(1);
});