chore: update dependencies and configs

packages/database/package.json

@@ -19,8 +19,8 @@
     "db:reset": "prisma db push --force-reset"
   },
   "dependencies": {
-    "@prisma/client": "^5.4.2",
     "@cloneweb/shared": "file:../shared",
+    "@prisma/client": "^5.4.2",
     "prisma": "^5.4.2"
   },
   "devDependencies": {
@@ -28,7 +28,7 @@
     "fast-check": "^3.13.2",
     "jest": "^29.7.0",
     "ts-jest": "^29.1.1",
-    "tsx": "^3.14.0",
+    "tsx": "^4.22.1",
     "typescript": "^5.2.2"
   }
-}
+}

services/api-gateway/package.json

@@ -13,31 +13,31 @@
     "test:watch": "jest --watch"
   },
   "dependencies": {
-    "@cloneweb/types": "file:../../packages/types",
     "@cloneweb/database": "file:../../packages/database",
     "@cloneweb/shared": "file:../../packages/shared",
-    "express": "^4.18.2",
+    "@cloneweb/types": "file:../../packages/types",
-    "cors": "^2.8.5",
-    "helmet": "^7.1.0",
-    "express-rate-limit": "^7.1.5",
-    "jsonwebtoken": "^9.0.2",
     "bcryptjs": "^2.4.3",
-    "zod": "^3.22.4",
+    "cors": "^2.8.5",
     "dotenv": "^16.3.1",
-    "winston": "^3.11.0"
+    "express": "^4.18.2",
+    "express-rate-limit": "^7.1.5",
+    "helmet": "^7.1.0",
+    "jsonwebtoken": "^9.0.2",
+    "winston": "^3.11.0",
+    "zod": "^3.22.4"
   },
   "devDependencies": {
-    "@types/express": "^4.17.21",
-    "@types/cors": "^2.8.17",
-    "@types/jsonwebtoken": "^9.0.5",
     "@types/bcryptjs": "^2.4.6",
+    "@types/cors": "^2.8.17",
+    "@types/express": "^4.17.21",
     "@types/jest": "^29.5.5",
+    "@types/jsonwebtoken": "^9.0.5",
     "@types/supertest": "^2.0.16",
     "fast-check": "^3.13.2",
     "jest": "^29.7.0",
     "supertest": "^6.3.3",
     "ts-jest": "^29.1.1",
-    "tsx": "^3.14.0",
+    "tsx": "^4.22.1",
     "typescript": "^5.2.2"
   }
-}
+}

services/crawler/package.json

@@ -11,27 +11,29 @@
     "test:watch": "jest --watch"
   },
   "dependencies": {
+    "@bull-board/api": "^5.10.2",
+    "@bull-board/express": "^5.10.2",
     "@cloneweb/database": "file:../../packages/database",
     "@cloneweb/shared": "file:../../packages/shared",
     "@cloneweb/types": "file:../../packages/types",
-    "express": "^4.18.2",
-    "puppeteer": "^21.5.2",
     "bull": "^4.12.2",
-    "@bull-board/api": "^5.10.2",
-    "@bull-board/express": "^5.10.2",
-    "ioredis": "^5.3.2",
-    "robots-parser": "^3.0.1",
-    "url-parse": "^1.5.10",
     "cheerio": "^1.0.0-rc.12",
-    "fast-check": "^3.15.0"
+    "express": "^4.18.2",
+    "fast-check": "^3.15.0",
+    "ioredis": "^5.3.2",
+    "playwright": "^1.60.0",
+    "playwright-extra": "^4.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2",
+    "robots-parser": "^3.0.1",
+    "url-parse": "^1.5.10"
   },
   "devDependencies": {
     "@types/express": "^4.17.21",
+    "@types/jest": "^29.5.8",
     "@types/node": "^20.10.4",
     "@types/url-parse": "^1.4.11",
-    "typescript": "^5.3.3",
-    "ts-node-dev": "^2.0.0",
     "jest": "^29.7.0",
-    "@types/jest": "^29.5.8"
+    "ts-node-dev": "^2.0.0",
+    "typescript": "^5.3.3"
   }
-}
+}

services/crawler/src/core/IntelligentCrawler.ts

@@ -1,4 +1,4 @@
-import puppeteer, { Browser, Page } from 'puppeteer';
+import { chromium, Browser, Page, BrowserContext } from 'playwright';
 import { URL } from 'url';
 import * as cheerio from 'cheerio';
 import robotsParser from 'robots-parser';
@@ -13,8 +13,8 @@ export class IntelligentCrawler {
   private lastRequestTime = 0;
 
   async initialize(): Promise<void> {
-    this.browser = await puppeteer.launch({
+    this.browser = await chromium.launch({
-      headless: 'new',
+      headless: true,
       args: [
         '--no-sandbox',
         '--disable-setuid-sandbox',
@@ -26,7 +26,7 @@ export class IntelligentCrawler {
         '--disable-gpu'
       ]
     });
-    logger.info('Puppeteer browser initialized');
+    logger.info('Playwright browser initialized');
   }
 
   async crawlWebsite(startUrl: string, config: CrawlerConfig): Promise<SiteMap> {
@@ -89,18 +89,18 @@ export class IntelligentCrawler {
   }
 
   async extractPageInfo(url: string, depth: number, config: CrawlerConfig): Promise<PageInfo | null> {
-    const page = await this.browser!.newPage();
+    const context = await this.browser!.newContext({
+      viewport: config.viewport,
+      userAgent: config.userAgent
+    });
+    const page = await context.newPage();
     
     try {
-      // Set viewport and user agent
-      await page.setViewport(config.viewport);
-      await page.setUserAgent(config.userAgent);
-      
       const startTime = Date.now();
       
       // Navigate to page
       const response = await page.goto(url, {
-        waitUntil: 'networkidle2',
+        waitUntil: 'networkidle',
         timeout: config.timeout
       });
 
@@ -162,7 +162,7 @@ export class IntelligentCrawler {
       };
       
     } finally {
-      await page.close();
+      await context.close();
     }
   }
 
@@ -224,7 +224,8 @@ export class IntelligentCrawler {
 
     try {
       const robotsUrl = `${origin}/robots.txt`;
-      const page = await this.browser!.newPage();
+      const context = await this.browser!.newContext();
+      const page = await context.newPage();
       
       const response = await page.goto(robotsUrl, { timeout: 10000 });
       
@@ -235,7 +236,7 @@ export class IntelligentCrawler {
         logger.info(`Loaded robots.txt for ${origin}`);
       }
       
-      await page.close();
+      await context.close();
     } catch (error) {
       logger.warn(`Failed to load robots.txt for ${origin}:`, error);
       this.robotsCache.set(origin, null);

simple-crawler/package.json

@@ -9,9 +9,11 @@
     "test": "echo \"Test with curl command\""
   },
   "dependencies": {
-    "axios": "^1.13.2",
+    "axios": "^1.16.1",
     "cheerio": "^1.0.0-rc.12",
     "express": "^4.18.2",
-    "puppeteer": "^21.5.2"
+    "playwright": "^1.60.0",
+    "playwright-extra": "^4.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2"
   }
-}
+}

simple-crawler/smart-cloner.js

@@ -1,7 +1,8 @@
 const fs = require('fs');
 const path = require('path');
-const puppeteer = require('puppeteer');
+const { chromium } = require('playwright');
 const { URL } = require('url');
+const aiService = require('./ai-recreation');
 
 class SmartCloner {
   constructor() {
@@ -84,14 +85,15 @@ class SmartCloner {
       const siteDir = path.join(this.baseOutputDir, `${siteName}_${timestamp}`);
       this.ensureDirectoryExists(siteDir);
 
-      browser = await puppeteer.launch({
+      browser = await chromium.launch({
-        headless: 'new',
+        headless: true,
         args: ['--no-sandbox', '--disable-setuid-sandbox', '--window-size=1920,1080']
       });
 
       const pagesToVisit = [url];
       const visitedPages = new Set();
       const pageMap = new Map(); // URL -> LocalFilename
+      const apiMocks = []; // Array of { url, method, status, headers, body }
 
       // Pre-seed the map so links can be rewritten to files we haven't visited yet
       pageMap.set(url.replace(/\/$/, ''), 'index.html');
@@ -108,16 +110,36 @@ class SmartCloner {
 
         console.log(`📄 [${pageCount}/${maxPages}] Clonando: ${currentUrl}`);
 
-        const page = await browser.newPage();
+        const context = await browser.newContext({ viewport: { width: 1920, height: 1080 }});
-        await page.setViewport({ width: 1920, height: 1080 });
+        const page = await context.newPage();
 
-        // Asset interception
+        // Asset and API interception
         page.on('response', async (response) => {
           const rUrl = response.url();
-          if (rUrl.startsWith('data:') || !response.ok() || response.request().method() !== 'GET') return;
+          const req = response.request();
+          
+          if (rUrl.startsWith('data:') || !response.ok()) return;
+          
+          const resourceType = req.resourceType();
+          if (resourceType === 'fetch' || resourceType === 'xhr') {
+            try {
+              const bodyStr = await response.text();
+              apiMocks.push({
+                url: rUrl,
+                method: req.method(),
+                status: response.status(),
+                headers: response.headers(),
+                body: bodyStr
+              });
+            } catch (e) { }
+            return;
+          }
+          
+          if (req.method() !== 'GET') return;
           try {
-            const contentType = response.headers()['content-type'];
+            const headers = response.headers();
-            const buffer = await response.buffer();
+            const contentType = headers['content-type'];
+            const buffer = await response.body();
             const localPath = this.generateLocalPath(rUrl, contentType, siteDir);
             this.ensureDirectoryExists(path.dirname(localPath));
             fs.writeFileSync(localPath, buffer);
@@ -126,10 +148,10 @@ class SmartCloner {
         });
 
         try {
-          await page.goto(currentUrl, { waitUntil: 'networkidle2', timeout: 60000 });
+          await page.goto(currentUrl, { waitUntil: 'networkidle', timeout: 60000 });
         } catch (e) {
           console.log(`⚠️ Falha ao carregar ${currentUrl}: ${e.message}`);
-          await page.close();
+          await context.close();
           continue;
         }
 
@@ -212,16 +234,104 @@ class SmartCloner {
             })();
           `;
           document.body.appendChild(script);
+
+          // 4. Inject Service Worker Registration
+          const swScript = document.createElement('script');
+          swScript.textContent = `
+            if ('serviceWorker' in navigator) {
+              window.addEventListener('load', function() {
+                navigator.serviceWorker.register('/mock-sw.js').then(function(registration) {
+                  console.log('Mock ServiceWorker registration successful with scope: ', registration.scope);
+                }, function(err) {
+                  console.log('Mock ServiceWorker registration failed: ', err);
+                });
+              });
+            }
+          `;
+          document.body.appendChild(swScript);
         }, resMapArr, pagMapArr);
 
+        // 5. AI Recreation of forms and dynamic content
+        const dynamicElements = await page.evaluate(() => {
+          return Array.from(document.querySelectorAll('form, [data-dynamic], .dynamic-content')).map(f => ({
+            html: f.outerHTML,
+            tagName: f.tagName
+          }));
+        });
+
+        for (const el of dynamicElements) {
+          const newHtml = await aiService.recreateComponent(el.html, el.tagName);
+          if (newHtml) {
+            await page.evaluate(({newHtml, oldHtml}) => {
+              const nodes = Array.from(document.querySelectorAll('form, [data-dynamic], .dynamic-content'));
+              const target = nodes.find(n => n.outerHTML === oldHtml);
+              if (target) {
+                target.outerHTML = newHtml;
+              }
+            }, { newHtml, oldHtml: el.html });
+          }
+        }
+
+        // 6. Expand Shadow DOM to Declarative Shadow DOM
+        await page.evaluate(() => {
+          function expandShadowRoots(node) {
+            if (node.shadowRoot) {
+              const template = document.createElement('template');
+              template.setAttribute('shadowrootmode', 'open');
+              template.innerHTML = node.shadowRoot.innerHTML;
+              node.insertBefore(template, node.firstChild);
+              // Recursively process inside shadowRoot
+              Array.from(node.shadowRoot.querySelectorAll('*')).forEach(expandShadowRoots);
+            }
+            Array.from(node.children).forEach(expandShadowRoots);
+          }
+          expandShadowRoots(document.body);
+        });
+
         const html = await page.content();
         const localFile = pageMap.get(cleanCurrentUrl) || 'index.html';
         fs.writeFileSync(path.join(siteDir, localFile), html);
-        await page.close();
+        await context.close();
       }
 
       const info = { url, timestamp: new Date().toISOString(), pages: pageCount, directory: siteDir };
       fs.writeFileSync(path.join(siteDir, 'clone-info.json'), JSON.stringify(info, null, 2));
+      
+      // Save Mock DB
+      fs.writeFileSync(path.join(siteDir, 'mock-db.json'), JSON.stringify(apiMocks, null, 2));
+      
+      // Create Service Worker
+      const swCode = `
+const MOCK_DB_URL = '/mock-db.json';
+let mockDb = null;
+
+self.addEventListener('install', (event) => {
+  self.skipWaiting();
+});
+
+self.addEventListener('activate', (event) => {
+  event.waitUntil(clients.claim());
+  event.waitUntil(
+    fetch(MOCK_DB_URL).then(res => res.json()).then(data => { mockDb = data; })
+  );
+});
+
+self.addEventListener('fetch', (event) => {
+  if (event.request.method === 'GET' && !event.request.url.includes('/mock-db.json') && mockDb) {
+    const mock = mockDb.find(m => m.url === event.request.url || event.request.url.includes(m.url));
+    if (mock) {
+      event.respondWith(
+        new Response(mock.body, {
+          status: mock.status,
+          headers: { 'Content-Type': mock.headers['content-type'] || 'application/json' }
+        })
+      );
+      return;
+    }
+  }
+});
+`;
+      fs.writeFileSync(path.join(siteDir, 'mock-sw.js'), swCode);
 
       return { success: true, path: siteDir, pagesCloned: pageCount };
     } catch (e) {