On this page
article
Puppeteer
Puppeteer
import puppeteer, {Browser} from "puppeteer";
const browser = await puppeteer.launch({
headless: true,
defaultViewport: {width: 1920, height: 1080},
userDataDir: "./userDir",
})
const page = await browser.newPage();
await page.goto("https://example.com", {waitUntil: "networkidle0"})
await page.waitForSelector("html");
const products: ProductT[] = await page.$$eval("#product-list .product", el =>
el.map(ele => {
return {
name: ele.querySelector(".product-name")?.textContent,
} as ProductT;
})
)
await page.locator("#load-more-button").click();
await page.waitForSelector(".delayed-item");
const delayedItems = await cheerioSection(await page.content())
Cheerio
import * as cheerio from "cheerio";
const res = await axios.get("https://example.com");
const $ = cheerio.load(res.data);
const $ = cheerio.load(pageContent);
const delayedItems: string[] = $("#delayed-content .delayed-item").map((_, el) => $(el).text()).get()
const $ = cheerio.load(pageContent)
const productItems: string[] = $("#product-list .product-name").map((_, el) => {
const match = $(el).text()
return match.trim()
}).get()
Testing
Vitest
Test loading a page and seeing if a selector works. Also a login page (used with directus in this case)
import { describe, expect, it, afterAll } from "vitest";
import puppeteer, { Browser, Page } from "puppeteer";
const username = "theusername";
const password = "thepassword";
const urlPrefix = "http://localhost:8055/admin";
let browser: Browser;
let page: Page;
afterAll(async (): Promise<void> => {
await page.close();
await browser.close();
})
async function getPage(): Promise<Page> {
if(!browser) {
browser = await puppeteer.launch({
headless: true,
defaultViewport: { width: 1920, height: 1080 },
userDataDir: "./userTestDir",
})
page = await browser.newPage();
}
await page.goto(urlPrefix, { waitUntil: "networkidle0" });
if (await page.$("input[type='email']") !== null) {
await page.type("input[type='email']", username);
await page.type("input[type='password']", password);
await page.click("div.sign-in button");
console.log("Logged in")
} else if (await page.$("look for continue button") !== null) {
} else {
console.log("Already signed in");
}
return page;
}
describe("Website Loading", async (): Promise<void> => {
const page = await getPage();
it('Should load the Home page', async (): Promise<void> => {
await page.goto(`${urlPrefix}/account/`, { waitUntil: "networkidle0"})
const homeTitle = await page.$eval("#welcome-section h2", e => e.textContent)
expect(homeTitle).toEqual("Demo Text I'm a h2");
}, 5000)
}, 10000);