I'm utilizing the Sparticuz/chromium package along with puppeteer-core for web scraping. Additionally, I've included a layer following the instructions outlined here. The code executes successfully when run locally, but upon deployment to AWS, it encounters the following error related to the directory "/var/task/src/functions/bin".
import chromium from "@sparticuz/chromium";
import dotenv from "dotenv";
import puppeteer from "puppeteer-core";
export const handler = async () => {
try {
const browser = await puppeteer.launch({
args: chromium.args,
defaultViewport: chromium.defaultViewport,
executablePath: await chromium.executablePath(),
headless: chromium.headless as boolean,
ignoreHTTPSErrors: true,
});
// const page = await browser.newPage();
const page = await browser.newPage();
await page.goto("https://shippos.nigerianports.gov.ng/", {
waitUntil: "domcontentloaded",
timeout: 120_000,
});
const element = await page.waitForSelector("#Apapa2_next");
const result = await element.evaluate((node) => node.textContent);
console.log(result);
console.log("Chromium:", await browser.version());
console.log("Page Title:", await page.title());
await page.close();
await browser.close();
await page.close();
await browser.close();
// await scraperVoceanLineups();
// await getNigeriaPortsLineups();
// await getMaputoVesselLineups();
return {
statusCode: 200,
body: `Urls succesfully scraped`,
};
} catch (error) {
return {
statusCode: 500,
body: `An error occured ${error.message}. Emails were sent to the relevant admins.`,
};
}
};