Puppeteer

  import puppeteer, {Browser} from "puppeteer";
  
  const browser = await puppeteer.launch({  
    headless: true,  
    defaultViewport: {width: 1920, height: 1080},  
    userDataDir: "./userDir",  
})  
const page = await browser.newPage();  
await page.goto("https://example.com", {waitUntil: "networkidle0"})  
await page.waitForSelector("html");
  
  const products: ProductT[] = await page.$$eval("#product-list .product", el =>  
    el.map(ele => {  
        return {  
            name: ele.querySelector(".product-name")?.textContent,  
        } as ProductT;  
    })  
)  
  
await page.locator("#load-more-button").click();  
await page.waitForSelector(".delayed-item");  
  
const delayedItems = await cheerioSection(await page.content())
  

Cheerio

  import * as cheerio from "cheerio";
  
  const res = await axios.get("https://example.com");
const $ = cheerio.load(res.data);
  
  const $ = cheerio.load(pageContent);
const delayedItems: string[] = $("#delayed-content .delayed-item").map((_, el) => $(el).text()).get()
  
  const $ = cheerio.load(pageContent)  
const productItems: string[] = $("#product-list .product-name").map((_, el) => {  
    const match = $(el).text()  
    return match.trim()  
}).get()
  

Testing

Vitest

Test loading a page and seeing if a selector works. Also a login page (used with directus in this case)

  import { describe, expect, it, afterAll } from "vitest";
import puppeteer, { Browser, Page } from "puppeteer";

const username = "theusername";
const password = "thepassword";

const urlPrefix = "http://localhost:8055/admin";

let browser: Browser;
let page: Page;

afterAll(async (): Promise<void> => {
    await page.close();
    await browser.close();
})

async function getPage(): Promise<Page> {
        if(!browser) {
            browser = await puppeteer.launch({
                headless: true,
                defaultViewport: { width: 1920, height: 1080 },
                userDataDir: "./userTestDir",
            })
            page = await browser.newPage();
        }
        await page.goto(urlPrefix, { waitUntil: "networkidle0" });
        if (await page.$("input[type='email']") !== null) {
            await page.type("input[type='email']", username);
            await page.type("input[type='password']", password);
            await page.click("div.sign-in button");
            console.log("Logged in")
        } else if (await page.$("look for continue button") !== null) {
        } else {
            console.log("Already signed in");
        }
    return page;
}

describe("Website Loading", async (): Promise<void> => {
    const page = await getPage();

    it('Should load the Home page', async (): Promise<void> => {
        await page.goto(`${urlPrefix}/account/`, { waitUntil: "networkidle0"})

        const homeTitle = await page.$eval("#welcome-section h2", e => e.textContent)
        expect(homeTitle).toEqual("Demo Text I'm a h2");
    }, 5000)

}, 10000);