chore: update dependencies and configs

services/crawler/src/core/IntelligentCrawler.ts

@@ -1,4 +1,4 @@
-import puppeteer, { Browser, Page } from 'puppeteer';
+import { chromium, Browser, Page, BrowserContext } from 'playwright';
 import { URL } from 'url';
 import * as cheerio from 'cheerio';
 import robotsParser from 'robots-parser';
@@ -13,8 +13,8 @@ export class IntelligentCrawler {
   private lastRequestTime = 0;
 
   async initialize(): Promise<void> {
-    this.browser = await puppeteer.launch({
-      headless: 'new',
+    this.browser = await chromium.launch({
+      headless: true,
       args: [
         '--no-sandbox',
         '--disable-setuid-sandbox',
@@ -26,7 +26,7 @@ export class IntelligentCrawler {
         '--disable-gpu'
       ]
     });
-    logger.info('Puppeteer browser initialized');
+    logger.info('Playwright browser initialized');
   }
 
   async crawlWebsite(startUrl: string, config: CrawlerConfig): Promise<SiteMap> {
@@ -89,18 +89,18 @@ export class IntelligentCrawler {
   }
 
   async extractPageInfo(url: string, depth: number, config: CrawlerConfig): Promise<PageInfo | null> {
-    const page = await this.browser!.newPage();
+    const context = await this.browser!.newContext({
+      viewport: config.viewport,
+      userAgent: config.userAgent
+    });
+    const page = await context.newPage();
     
     try {
-      // Set viewport and user agent
-      await page.setViewport(config.viewport);
-      await page.setUserAgent(config.userAgent);
-      
       const startTime = Date.now();
       
       // Navigate to page
       const response = await page.goto(url, {
-        waitUntil: 'networkidle2',
+        waitUntil: 'networkidle',
         timeout: config.timeout
       });
 
@@ -162,7 +162,7 @@ export class IntelligentCrawler {
       };
       
     } finally {
-      await page.close();
+      await context.close();
     }
   }
 
@@ -224,7 +224,8 @@ export class IntelligentCrawler {
 
     try {
       const robotsUrl = `${origin}/robots.txt`;
-      const page = await this.browser!.newPage();
+      const context = await this.browser!.newContext();
+      const page = await context.newPage();
       
       const response = await page.goto(robotsUrl, { timeout: 10000 });
       
@@ -235,7 +236,7 @@ export class IntelligentCrawler {
         logger.info(`Loaded robots.txt for ${origin}`);
       }
       
-      await page.close();
+      await context.close();
     } catch (error) {
       logger.warn(`Failed to load robots.txt for ${origin}:`, error);
       this.robotsCache.set(origin, null);