chore: update dependencies and configs

services/api-gateway/package.json

@@ -13,31 +13,31 @@
     "test:watch": "jest --watch"
   },
   "dependencies": {
-    "@cloneweb/types": "file:../../packages/types",
     "@cloneweb/database": "file:../../packages/database",
     "@cloneweb/shared": "file:../../packages/shared",
-    "express": "^4.18.2",
-    "cors": "^2.8.5",
-    "helmet": "^7.1.0",
-    "express-rate-limit": "^7.1.5",
-    "jsonwebtoken": "^9.0.2",
+    "@cloneweb/types": "file:../../packages/types",
     "bcryptjs": "^2.4.3",
-    "zod": "^3.22.4",
+    "cors": "^2.8.5",
     "dotenv": "^16.3.1",
-    "winston": "^3.11.0"
+    "express": "^4.18.2",
+    "express-rate-limit": "^7.1.5",
+    "helmet": "^7.1.0",
+    "jsonwebtoken": "^9.0.2",
+    "winston": "^3.11.0",
+    "zod": "^3.22.4"
   },
   "devDependencies": {
-    "@types/express": "^4.17.21",
-    "@types/cors": "^2.8.17",
-    "@types/jsonwebtoken": "^9.0.5",
     "@types/bcryptjs": "^2.4.6",
+    "@types/cors": "^2.8.17",
+    "@types/express": "^4.17.21",
     "@types/jest": "^29.5.5",
+    "@types/jsonwebtoken": "^9.0.5",
     "@types/supertest": "^2.0.16",
     "fast-check": "^3.13.2",
     "jest": "^29.7.0",
     "supertest": "^6.3.3",
     "ts-jest": "^29.1.1",
-    "tsx": "^3.14.0",
+    "tsx": "^4.22.1",
     "typescript": "^5.2.2"
   }
-}
+}

services/crawler/package.json

@@ -11,27 +11,29 @@
     "test:watch": "jest --watch"
   },
   "dependencies": {
+    "@bull-board/api": "^5.10.2",
+    "@bull-board/express": "^5.10.2",
     "@cloneweb/database": "file:../../packages/database",
     "@cloneweb/shared": "file:../../packages/shared",
     "@cloneweb/types": "file:../../packages/types",
-    "express": "^4.18.2",
-    "puppeteer": "^21.5.2",
     "bull": "^4.12.2",
-    "@bull-board/api": "^5.10.2",
-    "@bull-board/express": "^5.10.2",
-    "ioredis": "^5.3.2",
-    "robots-parser": "^3.0.1",
-    "url-parse": "^1.5.10",
     "cheerio": "^1.0.0-rc.12",
-    "fast-check": "^3.15.0"
+    "express": "^4.18.2",
+    "fast-check": "^3.15.0",
+    "ioredis": "^5.3.2",
+    "playwright": "^1.60.0",
+    "playwright-extra": "^4.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2",
+    "robots-parser": "^3.0.1",
+    "url-parse": "^1.5.10"
   },
   "devDependencies": {
     "@types/express": "^4.17.21",
+    "@types/jest": "^29.5.8",
     "@types/node": "^20.10.4",
     "@types/url-parse": "^1.4.11",
-    "typescript": "^5.3.3",
-    "ts-node-dev": "^2.0.0",
     "jest": "^29.7.0",
-    "@types/jest": "^29.5.8"
+    "ts-node-dev": "^2.0.0",
+    "typescript": "^5.3.3"
   }
-}
+}

services/crawler/src/core/IntelligentCrawler.ts

@@ -1,4 +1,4 @@
-import puppeteer, { Browser, Page } from 'puppeteer';
+import { chromium, Browser, Page, BrowserContext } from 'playwright';
 import { URL } from 'url';
 import * as cheerio from 'cheerio';
 import robotsParser from 'robots-parser';
@@ -13,8 +13,8 @@ export class IntelligentCrawler {
   private lastRequestTime = 0;
 
   async initialize(): Promise<void> {
-    this.browser = await puppeteer.launch({
-      headless: 'new',
+    this.browser = await chromium.launch({
+      headless: true,
       args: [
         '--no-sandbox',
         '--disable-setuid-sandbox',
@@ -26,7 +26,7 @@ export class IntelligentCrawler {
         '--disable-gpu'
       ]
     });
-    logger.info('Puppeteer browser initialized');
+    logger.info('Playwright browser initialized');
   }
 
   async crawlWebsite(startUrl: string, config: CrawlerConfig): Promise<SiteMap> {
@@ -89,18 +89,18 @@ export class IntelligentCrawler {
   }
 
   async extractPageInfo(url: string, depth: number, config: CrawlerConfig): Promise<PageInfo | null> {
-    const page = await this.browser!.newPage();
+    const context = await this.browser!.newContext({
+      viewport: config.viewport,
+      userAgent: config.userAgent
+    });
+    const page = await context.newPage();
     
     try {
-      // Set viewport and user agent
-      await page.setViewport(config.viewport);
-      await page.setUserAgent(config.userAgent);
-      
       const startTime = Date.now();
       
       // Navigate to page
       const response = await page.goto(url, {
-        waitUntil: 'networkidle2',
+        waitUntil: 'networkidle',
         timeout: config.timeout
       });
 
@@ -162,7 +162,7 @@ export class IntelligentCrawler {
       };
       
     } finally {
-      await page.close();
+      await context.close();
     }
   }
 
@@ -224,7 +224,8 @@ export class IntelligentCrawler {
 
     try {
       const robotsUrl = `${origin}/robots.txt`;
-      const page = await this.browser!.newPage();
+      const context = await this.browser!.newContext();
+      const page = await context.newPage();
       
       const response = await page.goto(robotsUrl, { timeout: 10000 });
       
@@ -235,7 +236,7 @@ export class IntelligentCrawler {
         logger.info(`Loaded robots.txt for ${origin}`);
       }
       
-      await page.close();
+      await context.close();
     } catch (error) {
       logger.warn(`Failed to load robots.txt for ${origin}:`, error);
       this.robotsCache.set(origin, null);