jobs-monorepo/glassdoor-scraper/services/browserService.ts

import { Page } from "puppeteer-core";
import { randomDelay } from "../utils/browser";

export interface JobListing {
  title: string;
  company: string;
  location: string;
  description: string;
  skills: string[];
  jobLink: string;
  companyLink: string;
  age: string;
  jobId: string;
  provider: number;
}

export class BrowserService {
  private page: Page;

  constructor(page: Page) {
    this.page = page;
  }

  async searchJobs(keywords: string, location: string) {
    try {
      // Wait for search form to load
      const keywordInput = await this.page.waitForSelector(
        "#searchBar-jobTitle, input[name='sc.keyword'], input[placeholder*='Job title']",
        { timeout: 10000 }
      );

      if (keywordInput) {
        await keywordInput.click({ clickCount: 3 });
        await keywordInput.type(keywords, { delay: 130 });
        await randomDelay(1000, 2000);
      }

      const locationInput = await this.page.waitForSelector(
        "#searchBar-location, input[name='locKeyword'], input[placeholder*='Location']",
        { timeout: 5000 }
      );

      if (locationInput) {
        await locationInput.click({ clickCount: 3 });
        await locationInput.type(location, { delay: 130 });
        await randomDelay(1000, 2000);
        await locationInput.press("Enter");

        // Wait for search results to load
        await this.page.waitForNavigation({
          waitUntil: "networkidle2",
          timeout: 30000,
        });
      }
    } catch (error) {
      console.log(
        "Search form not found, attempting to extract jobs from current page..."
      );
    }
  }

  async extractJobListings(): Promise<JobListing[]> {
    // Wait for job listings to appear
    try {
      await this.page.waitForSelector(
        'li[data-test="jobListing"], .JobsList_jobListItem__wjTHv, [data-jobid]',
        { timeout: 10000 }
      );
    } catch (error) {
      console.log("No job listings found on the page");
      return [];
    }

    // Extract job data using page.evaluate
    const jobs = await this.page.evaluate(() => {
      // Multiple selectors to handle different Glassdoor layouts
      const jobElements = document.querySelectorAll(`
        li[data-test="jobListing"],
        .JobsList_jobListItem__wjTHv,
        li[data-jobid],
        .react-job-listing
      `);

      const jobs: any[] = [];

      jobElements.forEach((element, index) => {
        try {
          // Extract job ID
          const jobId =
            element.getAttribute("data-jobid") ||
            element.getAttribute("data-brandviews")?.match(/jlid=(\d+)/)?.[1] ||
            `job-${index + 1}`;

          // Extract job title and link
          const titleElement = element.querySelector(`
            a[data-test="job-title"],
            .JobCard_jobTitle__GLyJ1,
            .jobLink,
            a[id*="job-title"]
          `) as HTMLAnchorElement;

          // Extract company name and link
          const companyElement = element.querySelector(`
            .EmployerProfile_compactEmployerName__9MGcV,
            [data-test="employer-name"] a,
            .employerName,
            .EmployerProfile_employerNameContainer__ptolz span
          `) as HTMLElement;

          // Extract location
          const locationElement = element.querySelector(`
            .JobCard_location__Ds1fM,
            [data-test="emp-location"],
            [data-test="job-location"],
            .location,
            [id*="job-location"]
          `) as HTMLElement;

          // Extract job description
          const descriptionElement = element.querySelector(`
            .JobCard_jobDescriptionSnippet__l1tnl,
            [data-test="descSnippet"],
            .jobDescriptionSnippet
          `) as HTMLElement;

          // Extract job age
          const ageElement = element.querySelector(`
            .JobCard_listingAge__jJsuc,
            [data-test="job-age"],
            .listingAge
          `) as HTMLElement;

          // Extract skills if available
          const skillsText = descriptionElement?.textContent || "";
          const skillsMatch = skillsText.match(
            /(?:Vaardigheden|Skills):\s*([^.]+)/i
          );
          const skills = skillsMatch
            ? skillsMatch[1]
                .split(",")
                .map((skill) => skill.trim())
                .filter((skill) => skill.length > 0)
            : [];

          if (titleElement && companyElement) {
            const title = titleElement.textContent?.trim() || "";
            const company = companyElement.textContent?.trim() || "";
            const location = locationElement?.textContent?.trim() || "";
            const description = descriptionElement?.textContent?.trim() || "";
            const age = ageElement?.textContent?.trim() || "";

            // Build full URLs for links
            const jobLink = titleElement.href
              ? titleElement.href.startsWith("http")
                ? titleElement.href
                : `https://www.glassdoor.com${titleElement.href}`
              : "";

            const companyLink =
              companyElement.tagName === "A"
                ? (companyElement as HTMLAnchorElement).href?.startsWith("http")
                  ? (companyElement as HTMLAnchorElement).href
                  : `https://www.glassdoor.com${
                      (companyElement as HTMLAnchorElement).href
                    }`
                : "";

            jobs.push({
              title,
              company,
              location,
              description: description.substring(0, 500), // Limit description length
              skills,
              jobLink,
              companyLink,
              age,
              jobId,
              provider: 2,
            });
          }
        } catch (error) {
          console.error(`Error extracting job ${index}:`, error);
        }
      });

      return jobs;
    });

    console.log(`Found ${jobs.length} jobs on Glassdoor`);
    jobs.forEach((job, index) => {
      console.log(
        `${index + 1}. ${job.title} at ${job.company} - ${job.location}`
      );
      if (job.skills.length > 0) {
        console.log(`   Skills: ${job.skills.join(", ")}`);
      }
    });

    return jobs;
  }
}