import puppeteer, { Browser, Page } from "puppeteer-core"; import { ChildProcess } from "child_process"; import * as dotenv from "dotenv"; import { ChromeVersionResponse, launchChromeWithDebugging, randomDelay, waitForChromeReady, } from "./utils/browser"; import { RabbitMQClient, SearchQuery, createRabbitMQClient, } from "./utils/rabbitmq"; import { BrowserService } from "./services/browserService"; import { saveJobs } from "./queries/save-jobs"; import { saveJobDescriptions } from "./queries/save-job-description"; if (dotenv.config({ path: ".local.env" }).error) { console.log("No .local.env file found, trying .env"); if (dotenv.config().error) { console.log("No .env file found, using system environment variables"); } } let globalBrowser: Browser | null = null; let chrome: ChildProcess; async function cleanup() { console.log("Starting cleanup process..."); if (rabbitMQClient) { console.log("Closing RabbitMQ connection"); await rabbitMQClient.close(); rabbitMQClient = null; } if (globalBrowser) { console.log("Closing browser on app shutdown"); await globalBrowser.close(); globalBrowser = null; } if (chrome) { chrome.kill("SIGINT"); } console.log("Cleanup completed"); } async function initializeBrowser(): Promise<{ browser: Browser; page: Page }> { chrome = launchChromeWithDebugging(); await waitForChromeReady(); const response = await fetch("http://localhost:9222/json/version", { headers: { Origin: "", }, }); if (!response.ok) { throw new Error("Failed to connect to Chrome debugging port"); } const chromeInstance = (await response.json()) as ChromeVersionResponse; const { webSocketDebuggerUrl } = chromeInstance; const browser = await puppeteer.connect({ browserWSEndpoint: webSocketDebuggerUrl, defaultViewport: null, }); globalBrowser = browser; console.log("Connected to browser successfully"); let pages = await browser.pages(); console.log(`Found ${pages.length} pages`); let existingcookies = await browser.cookies(); await browser.deleteCookie(...existingcookies); let page: Page; if (pages.length === 0) { console.log("No pages found, creating a new page"); page = await browser.newPage(); } else { page = pages[0]; } await page.setUserAgent( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" ); return { browser, page }; } async function scrapeGlassdoorJobs(searchQuery: SearchQuery): Promise { try { const { page } = await initializeBrowser(); const browserService = new BrowserService(page); const searchUrl = `https://www.glassdoor.com/Job/jobs.htm`; await page.goto(searchUrl, { waitUntil: ["networkidle2", "domcontentloaded"], timeout: 60000, }); await randomDelay(1500, 2500); await browserService.searchJobs(searchQuery.keywords, searchQuery.location); const jobs = await browserService.extractJobListings(searchQuery.location); if (jobs.length > 0) { await saveJobs(jobs); await saveJobDescriptions(jobs); console.log( `saved ${jobs.length} jobs from Glassdoor for query:`, searchQuery ); } else { console.log("No jobs found for this search query"); } await randomDelay(1500, 2500); } catch (error) { console.error("Error scraping Glassdoor jobs:", error); throw error; } if (globalBrowser) { console.log("Closing browser on app shutdown"); await globalBrowser.close(); globalBrowser = null; } if (chrome) { chrome.kill("SIGINT"); } } let rabbitMQClient: RabbitMQClient | null = null; async function subscribeToGlassdoorQueue(): Promise { try { rabbitMQClient = await createRabbitMQClient(); await rabbitMQClient.subscribe(async (searchQuery: SearchQuery) => { await scrapeGlassdoorJobs(searchQuery); }); } catch (error) { console.error("Failed to set up RabbitMQ subscription:", error); process.exit(1); } } async function main() { console.log("Starting Glassdoor scraper service..."); await subscribeToGlassdoorQueue(); } process.on("SIGINT", async () => { console.log("Received SIGINT signal"); await cleanup(); if (chrome) { chrome.kill("SIGINT"); } process.exit(0); }); process.on("SIGTERM", async () => { console.log("Received SIGTERM signal"); await cleanup(); if (chrome) { chrome.kill("SIGTERM"); } process.exit(0); }); main().catch((error) => { console.error("Fatal error:", error); process.exit(1); });