import puppeteer, { Browser, Page } from "puppeteer-core"; import { ChildProcess } from "child_process"; import * as dotenv from "dotenv"; import { ChromeVersionResponse, launchChromeWithDebugging, randomDelay, waitForChromeReady, } from "./utils/browser"; import { BrowserService } from "./services/browserService"; import { saveJob } from "./queries/save-jobs"; import { saveJobDescription } from "./queries/save-job-description"; import { Job } from "./types/job"; if (dotenv.config({ path: ".local.env" }).error) { console.log("No .local.env file found, trying .env"); if (dotenv.config().error) { console.log("No .env file found, using system environment variables"); } } let globalBrowser: Browser | null = null; let chrome: ChildProcess; async function cleanup() { console.log("Starting cleanup process..."); if (globalBrowser) { console.log("Closing browser on app shutdown"); await globalBrowser.close(); globalBrowser = null; } if (chrome) { chrome.kill("SIGINT"); } console.log("Cleanup completed"); } async function initializeBrowser(): Promise<{ browser: Browser; page: Page }> { chrome = launchChromeWithDebugging(); await waitForChromeReady(); const response = await fetch("http://localhost:9222/json/version", { headers: { Origin: "", }, }); if (!response.ok) { throw new Error("Failed to connect to Chrome debugging port"); } const chromeInstance = (await response.json()) as ChromeVersionResponse; const { webSocketDebuggerUrl } = chromeInstance; const browser = await puppeteer.connect({ browserWSEndpoint: webSocketDebuggerUrl, defaultViewport: null, }); globalBrowser = browser; console.log("Connected to browser successfully"); let pages = await browser.pages(); console.log(`Found ${pages.length} pages`); let existingcookies = await browser.cookies(); await browser.deleteCookie(...existingcookies); let page: Page; if (pages.length === 0) { console.log("No pages found, creating a new page"); page = await browser.newPage(); } else { page = pages[0]; } await page.setUserAgent( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" ); return { browser, page }; } async function scrapeHiringCafeJobs(): Promise { try { const { page } = await initializeBrowser(); const browserService = new BrowserService(page); const searchUrl = `https://hiring.cafe/?searchState=%7B%22defaultToUserLocation%22%3Afalse%2C%22searchQuery%22%3A%22software%22%2C%22dateFetchedPastNDays%22%3A2%7D`; await page.goto(searchUrl, { waitUntil: ["networkidle2", "domcontentloaded"], timeout: 60000, }); const searchElement = await page.waitForSelector("#query-search-v4", { timeout: 60000, }); if (searchElement != null) { console.log("found search element, continue"); } else { console.log("could not find search element, aborting"); return; } await page.waitForNetworkIdle({ idleTime: 2000, timeout: 0 }); let jobLinks: string[] = []; while (jobLinks.length < 1) { await randomDelay(1000, 2000); jobLinks = await browserService.getJobLinks(); console.log(`Found ${jobLinks.length} job links`); } for (const link of jobLinks) { try { const job = await browserService.getJobData(link); const jobId = await saveJob(job); if (jobId) { await saveJobDescription({ criteria: job.criteria, description: job.description || "", jobId: jobId, }); } await randomDelay(1000, 2000); } catch (error) { console.error(`Error processing job link ${link}:`, error); await randomDelay(1000, 2000); continue; } } console.log("Saved job listings to the database"); console.log("Saved job descriptions to the database"); await randomDelay(1500, 2500); } catch (error) { console.error("Error scraping hiring cafe jobs:", error); throw error; } if (globalBrowser) { console.log("Closing browser on app shutdown"); await globalBrowser.close(); globalBrowser = null; } if (chrome) { chrome.kill("SIGINT"); } } async function main() { console.log("Starting Hiring Cafe scraper service..."); await scrapeHiringCafeJobs(); } process.on("SIGINT", async () => { console.log("Received SIGINT signal"); await cleanup(); if (chrome) { chrome.kill("SIGINT"); } process.exit(0); }); process.on("SIGTERM", async () => { console.log("Received SIGTERM signal"); await cleanup(); if (chrome) { chrome.kill("SIGTERM"); } process.exit(0); }); main().catch((error) => { console.error("Fatal error:", error); process.exit(1); });