import puppeteer, { Browser, Page } from "puppeteer-core"; import { ChildProcess } from "child_process"; import * as dotenv from "dotenv"; import { ChromeVersionResponse, launchChromeWithDebugging, randomDelay, waitForChromeReady, } from "./utils/browser"; import { RabbitMQClient, SearchQuery, createRabbitMQClient, } from "./utils/rabbitmq"; import { BrowserService } from "./services/browserService"; dotenv.config(); let globalBrowser: Browser | null = null; let chrome: ChildProcess; async function cleanup() { console.log("Starting cleanup process..."); if (rabbitMQClient) { console.log("Closing RabbitMQ connection"); await rabbitMQClient.close(); rabbitMQClient = null; } if (globalBrowser) { console.log("Closing browser on app shutdown"); await globalBrowser.close(); globalBrowser = null; } if (chrome) { chrome.kill("SIGINT"); } console.log("Cleanup completed"); } async function initializeBrowser(): Promise<{ browser: Browser; page: Page }> { chrome = launchChromeWithDebugging(); await waitForChromeReady(); const response = await fetch("http://localhost:9222/json/version", { headers: { Origin: "", }, }); if (!response.ok) { throw new Error("Failed to connect to Chrome debugging port"); } const chromeInstance = (await response.json()) as ChromeVersionResponse; const { webSocketDebuggerUrl } = chromeInstance; const browser = await puppeteer.connect({ browserWSEndpoint: webSocketDebuggerUrl, defaultViewport: null, }); globalBrowser = browser; console.log("Connected to browser successfully"); let pages = await browser.pages(); console.log(`Found ${pages.length} pages`); let existingcookies = await browser.cookies(); await browser.deleteCookie(...existingcookies); let page: Page; if (pages.length === 0) { console.log("No pages found, creating a new page"); page = await browser.newPage(); } else { page = pages[0]; } await page.setUserAgent( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" ); return { browser, page }; } async function scrapeGlassdoorJobs(searchQuery: SearchQuery): Promise { console.log( `Starting Glassdoor scrape for: ${searchQuery.keywords} in ${searchQuery.location}` ); try { const { page } = await initializeBrowser(); const browserService = new BrowserService(page); // Build Glassdoor search URL const searchUrl = buildGlassdoorSearchURL(searchQuery); console.log(`Navigating to: ${searchUrl}`); await page.goto(searchUrl, { waitUntil: ["networkidle2", "domcontentloaded"], timeout: 60000, }); await randomDelay(1500, 2500); await browserService.searchJobs(searchQuery.keywords, searchQuery.location); const jobs = await browserService.extractJobListings(); console.log(`\n=== SCRAPING RESULTS ===`); console.log(`Successfully extracted ${jobs.length} jobs from Glassdoor`); if (jobs.length > 0) { console.log(`\n=== JOB LISTINGS ===`); jobs.forEach((job, index) => { console.log(`\n${index + 1}. ${job.title}`); console.log(` Company: ${job.company}`); console.log(` Location: ${job.location}`); console.log(` Age: ${job.age}`); console.log(` Job ID: ${job.jobId}`); if (job.skills.length > 0) { console.log(` Skills: ${job.skills.join(", ")}`); } if (job.jobLink) { console.log(` Job Link: ${job.jobLink}`); } if (job.description) { console.log( ` Description: ${job.description.substring(0, 200)}...` ); } }); // Here you can process the jobs further: // - Save to database // - Send to another queue // - Perform additional analysis console.log(`\n=== END RESULTS ===`); } else { console.log("No jobs found for this search query"); } await randomDelay(1500, 2500); } catch (error) { console.error("Error scraping Glassdoor jobs:", error); throw error; } } function getGlassdoorDomain(location: string): string { const locationLower = location.toLowerCase(); // Country-specific Glassdoor domains const countryDomains: { [key: string]: string } = { // Asia Pacific japan: "https://www.glassdoor.com/Job", // Glassdoor Japan uses .com singapore: "https://www.glassdoor.sg/Job", australia: "https://www.glassdoor.com.au/Job", "hong kong": "https://www.glassdoor.com/Job", // Uses .com with HK location india: "https://www.glassdoor.co.in/Job", // Europe "united kingdom": "https://www.glassdoor.co.uk/Job", uk: "https://www.glassdoor.co.uk/Job", germany: "https://www.glassdoor.de/Job", france: "https://www.glassdoor.fr/Job", netherlands: "https://www.glassdoor.nl/Job", belgium: "https://www.glassdoor.be/Job", austria: "https://www.glassdoor.at/Job", switzerland: "https://www.glassdoor.ch/Job", ireland: "https://www.glassdoor.ie/Job", spain: "https://www.glassdoor.es/Job", italy: "https://www.glassdoor.it/Job", // North America canada: "https://www.glassdoor.ca/Job", mexico: "https://www.glassdoor.com.mx/Job", // South America brazil: "https://www.glassdoor.com.br/Job", argentina: "https://www.glassdoor.com.ar/Job", }; // Check for country matches for (const [country, domain] of Object.entries(countryDomains)) { if (locationLower.includes(country)) { return domain; } } // Check for city-specific mappings const cityMappings: { [key: string]: string } = { tokyo: "https://www.glassdoor.com/Job", osaka: "https://www.glassdoor.com/Job", kyoto: "https://www.glassdoor.com/Job", london: "https://www.glassdoor.co.uk/Job", manchester: "https://www.glassdoor.co.uk/Job", berlin: "https://www.glassdoor.de/Job", munich: "https://www.glassdoor.de/Job", paris: "https://www.glassdoor.fr/Job", amsterdam: "https://www.glassdoor.nl/Job", toronto: "https://www.glassdoor.ca/Job", vancouver: "https://www.glassdoor.ca/Job", sydney: "https://www.glassdoor.com.au/Job", melbourne: "https://www.glassdoor.com.au/Job", singapore: "https://www.glassdoor.sg/Job", mumbai: "https://www.glassdoor.co.in/Job", bangalore: "https://www.glassdoor.co.in/Job", delhi: "https://www.glassdoor.co.in/Job", }; for (const [city, domain] of Object.entries(cityMappings)) { if (locationLower.includes(city)) { return domain; } } // Default to US site return "https://www.glassdoor.com/Job"; } function buildGlassdoorSearchURL(query: SearchQuery): string { const baseDomain = getGlassdoorDomain(query.location); const baseUrl = `${baseDomain}/jobs.htm`; console.log( `Selected Glassdoor domain for location "${query.location}": ${baseDomain}` ); const params = new URLSearchParams(); // if (query.Keywords) { // params.set("sc.keyword", query.Keywords); // } // if (query.Location) { // params.set("locT", "C"); // params.set("locId", query.Location); // } return `${baseUrl}?${params.toString()}`; } let rabbitMQClient: RabbitMQClient | null = null; async function subscribeToGlassdoorQueue(): Promise { try { rabbitMQClient = await createRabbitMQClient(); await rabbitMQClient.subscribe(async (searchQuery: SearchQuery) => { await scrapeGlassdoorJobs(searchQuery); }); } catch (error) { console.error("Failed to set up RabbitMQ subscription:", error); process.exit(1); } } async function main() { console.log("Starting Glassdoor scraper service..."); await subscribeToGlassdoorQueue(); } process.on("SIGINT", async () => { console.log("Received SIGINT signal"); await cleanup(); if (chrome) { chrome.kill("SIGINT"); } process.exit(0); }); process.on("SIGTERM", async () => { console.log("Received SIGTERM signal"); await cleanup(); if (chrome) { chrome.kill("SIGTERM"); } process.exit(0); }); main().catch((error) => { console.error("Fatal error:", error); process.exit(1); });