jobs-monorepo/services/scraper-hiringcafe/index.ts

import puppeteer, { Browser, Page } from "puppeteer-core";
import { ChildProcess } from "child_process";
import * as dotenv from "dotenv";
import {
  ChromeVersionResponse,
  launchChromeWithDebugging,
  randomDelay,
  waitForChromeReady,
} from "./utils/browser";
import { BrowserService } from "./services/browserService";
import { saveJob } from "./queries/save-jobs";
import { saveJobDescription } from "./queries/save-job-description";
import { Job } from "./types/job";

if (dotenv.config({ path: ".local.env" }).error) {
  console.log("No .local.env file found, trying .env");
  if (dotenv.config().error) {
    console.log("No .env file found, using system environment variables");
  }
}

let globalBrowser: Browser | null = null;
let chrome: ChildProcess;

async function cleanup() {
  console.log("Starting cleanup process...");

  if (globalBrowser) {
    console.log("Closing browser on app shutdown");
    await globalBrowser.close();
    globalBrowser = null;
  }

  if (chrome) {
    chrome.kill("SIGINT");
  }

  console.log("Cleanup completed");
}

async function initializeBrowser(): Promise<{ browser: Browser; page: Page }> {
  let port = 9223;
  chrome = launchChromeWithDebugging(port);
  await waitForChromeReady();

  const response = await fetch(`http://localhost:${port}/json/version`, {
    headers: {
      Origin: "",
    },
  });

  if (!response.ok) {
    throw new Error("Failed to connect to Chrome debugging port");
  }

  const chromeInstance = (await response.json()) as ChromeVersionResponse;
  const { webSocketDebuggerUrl } = chromeInstance;

  const browser = await puppeteer.connect({
    browserWSEndpoint: webSocketDebuggerUrl,
    defaultViewport: null,
  });

  globalBrowser = browser;
  console.log("Connected to browser successfully");

  let pages = await browser.pages();
  console.log(`Found ${pages.length} pages`);

  let existingcookies = await browser.cookies();
  await browser.deleteCookie(...existingcookies);

  let page: Page;
  if (pages.length === 0) {
    console.log("No pages found, creating a new page");
    page = await browser.newPage();
  } else {
    page = pages[0];
  }

  await page.setUserAgent(
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
  );

  return { browser, page };
}

async function scrapeHiringCafeJobs(): Promise<void> {
  try {
    const { page } = await initializeBrowser();
    const browserService = new BrowserService(page);

    const searchUrl = `https://hiring.cafe/?searchState=%7B%22defaultToUserLocation%22%3Afalse%2C%22searchQuery%22%3A%22software%22%2C%22dateFetchedPastNDays%22%3A2%7D`;

    await page.goto(searchUrl, {
      waitUntil: ["networkidle2", "domcontentloaded"],
      timeout: 60000,
    });

    const searchElement = await page.waitForSelector("#query-search-v4", {
      timeout: 60000,
    });

    if (searchElement != null) {
      console.log("found search element, continue");
    } else {
      console.log("could not find search element, aborting");
      return;
    }

    await page.waitForNetworkIdle({ idleTime: 2000, timeout: 0 });

    let jobLinks: string[] = [];

    while (jobLinks.length < 1) {
      await randomDelay(1000, 2000);
      jobLinks = await browserService.getJobLinks();
      console.log(`Found ${jobLinks.length} job links`);
    }

    for (const link of jobLinks) {
      try {
        const job = await browserService.getJobData(link);

        const jobId = await saveJob(job);

        if (jobId) {
          await saveJobDescription({
            criteria: job.criteria,
            description: job.description || "",
            jobId: jobId,
          });
        }
        await randomDelay(1000, 2000);
      } catch (error) {
        console.error(`Error processing job link ${link}:`, error);
        await randomDelay(1000, 2000);

        continue;
      }
    }

    console.log("Saved job listings to the database");

    console.log("Saved job descriptions to the database");

    await randomDelay(1500, 2500);
  } catch (error) {
    console.error("Error scraping hiring cafe jobs:", error);
    throw error;
  }

  if (globalBrowser) {
    console.log("Closing browser on app shutdown");
    await globalBrowser.close();
    globalBrowser = null;
  }

  if (chrome) {
    chrome.kill("SIGINT");
  }
}

async function main() {
  console.log("Starting Hiring Cafe scraper service...");
  await scrapeHiringCafeJobs();
}

process.on("SIGINT", async () => {
  console.log("Received SIGINT signal");
  await cleanup();
  if (chrome) {
    chrome.kill("SIGINT");
  }
  process.exit(0);
});

process.on("SIGTERM", async () => {
  console.log("Received SIGTERM signal");
  await cleanup();
  if (chrome) {
    chrome.kill("SIGTERM");
  }
  process.exit(0);
});

main().catch((error) => {
  console.error("Fatal error:", error);
  process.exit(1);
});