fix: Use puppeteer adblocker to block cookies notices

hoarder-app · Mar 5, 2024 · 591358c · 591358c
1 parent fdd60a5
commit 591358c
Show file tree

Hide file tree

Showing 3 changed files with 120 additions and 0 deletions.
diff --git a/packages/workers/crawler.ts b/packages/workers/crawler.ts
@@ -18,6 +18,7 @@ import { db } from "@hoarder/db";
 import { Browser } from "puppeteer";
 import puppeteer from "puppeteer-extra";
 import StealthPlugin from "puppeteer-extra-plugin-stealth";
+import AdblockerPlugin from "puppeteer-extra-plugin-adblocker";
 
 import metascraper from "metascraper";
 
@@ -70,6 +71,11 @@ async function launchBrowser() {
 export class CrawlerWorker {
   static async build() {
     puppeteer.use(StealthPlugin());
+    puppeteer.use(
+      AdblockerPlugin({
+        blockTrackersAndAnnoyances: true,
+      }),
+    );
     await launchBrowser();
 
     logger.info("Starting crawler worker ...");

diff --git a/packages/workers/package.json b/packages/workers/package.json
@@ -26,6 +26,7 @@
     "openai": "^4.26.1",
     "puppeteer": "^22.0.0",
     "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-adblocker": "^2.13.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
     "tsx": "^4.7.1",
     "typescript": "^5",

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml