chore: update dependencies and configs

simple-crawler/smart-cloner.js

@@ -1,7 +1,8 @@
 const fs = require('fs');
 const path = require('path');
-const puppeteer = require('puppeteer');
+const { chromium } = require('playwright');
 const { URL } = require('url');
+const aiService = require('./ai-recreation');
 
 class SmartCloner {
   constructor() {
@@ -84,14 +85,15 @@ class SmartCloner {
       const siteDir = path.join(this.baseOutputDir, `${siteName}_${timestamp}`);
       this.ensureDirectoryExists(siteDir);
 
-      browser = await puppeteer.launch({
-        headless: 'new',
+      browser = await chromium.launch({
+        headless: true,
         args: ['--no-sandbox', '--disable-setuid-sandbox', '--window-size=1920,1080']
       });
 
       const pagesToVisit = [url];
       const visitedPages = new Set();
       const pageMap = new Map(); // URL -> LocalFilename
+      const apiMocks = []; // Array of { url, method, status, headers, body }
 
       // Pre-seed the map so links can be rewritten to files we haven't visited yet
       pageMap.set(url.replace(/\/$/, ''), 'index.html');
@@ -108,16 +110,36 @@ class SmartCloner {
 
         console.log(`📄 [${pageCount}/${maxPages}] Clonando: ${currentUrl}`);
 
-        const page = await browser.newPage();
-        await page.setViewport({ width: 1920, height: 1080 });
+        const context = await browser.newContext({ viewport: { width: 1920, height: 1080 }});
+        const page = await context.newPage();
 
-        // Asset interception
+        // Asset and API interception
         page.on('response', async (response) => {
           const rUrl = response.url();
-          if (rUrl.startsWith('data:') || !response.ok() || response.request().method() !== 'GET') return;
+          const req = response.request();
+          
+          if (rUrl.startsWith('data:') || !response.ok()) return;
+          
+          const resourceType = req.resourceType();
+          if (resourceType === 'fetch' || resourceType === 'xhr') {
+            try {
+              const bodyStr = await response.text();
+              apiMocks.push({
+                url: rUrl,
+                method: req.method(),
+                status: response.status(),
+                headers: response.headers(),
+                body: bodyStr
+              });
+            } catch (e) { }
+            return;
+          }
+          
+          if (req.method() !== 'GET') return;
           try {
-            const contentType = response.headers()['content-type'];
-            const buffer = await response.buffer();
+            const headers = response.headers();
+            const contentType = headers['content-type'];
+            const buffer = await response.body();
             const localPath = this.generateLocalPath(rUrl, contentType, siteDir);
             this.ensureDirectoryExists(path.dirname(localPath));
             fs.writeFileSync(localPath, buffer);
@@ -126,10 +148,10 @@ class SmartCloner {
         });
 
         try {
-          await page.goto(currentUrl, { waitUntil: 'networkidle2', timeout: 60000 });
+          await page.goto(currentUrl, { waitUntil: 'networkidle', timeout: 60000 });
         } catch (e) {
           console.log(`⚠️ Falha ao carregar ${currentUrl}: ${e.message}`);
-          await page.close();
+          await context.close();
           continue;
         }
 
@@ -212,16 +234,104 @@ class SmartCloner {
             })();
           `;
           document.body.appendChild(script);
+
+          // 4. Inject Service Worker Registration
+          const swScript = document.createElement('script');
+          swScript.textContent = `
+            if ('serviceWorker' in navigator) {
+              window.addEventListener('load', function() {
+                navigator.serviceWorker.register('/mock-sw.js').then(function(registration) {
+                  console.log('Mock ServiceWorker registration successful with scope: ', registration.scope);
+                }, function(err) {
+                  console.log('Mock ServiceWorker registration failed: ', err);
+                });
+              });
+            }
+          `;
+          document.body.appendChild(swScript);
         }, resMapArr, pagMapArr);
 
+        // 5. AI Recreation of forms and dynamic content
+        const dynamicElements = await page.evaluate(() => {
+          return Array.from(document.querySelectorAll('form, [data-dynamic], .dynamic-content')).map(f => ({
+            html: f.outerHTML,
+            tagName: f.tagName
+          }));
+        });
+
+        for (const el of dynamicElements) {
+          const newHtml = await aiService.recreateComponent(el.html, el.tagName);
+          if (newHtml) {
+            await page.evaluate(({newHtml, oldHtml}) => {
+              const nodes = Array.from(document.querySelectorAll('form, [data-dynamic], .dynamic-content'));
+              const target = nodes.find(n => n.outerHTML === oldHtml);
+              if (target) {
+                target.outerHTML = newHtml;
+              }
+            }, { newHtml, oldHtml: el.html });
+          }
+        }
+
+        // 6. Expand Shadow DOM to Declarative Shadow DOM
+        await page.evaluate(() => {
+          function expandShadowRoots(node) {
+            if (node.shadowRoot) {
+              const template = document.createElement('template');
+              template.setAttribute('shadowrootmode', 'open');
+              template.innerHTML = node.shadowRoot.innerHTML;
+              node.insertBefore(template, node.firstChild);
+              // Recursively process inside shadowRoot
+              Array.from(node.shadowRoot.querySelectorAll('*')).forEach(expandShadowRoots);
+            }
+            Array.from(node.children).forEach(expandShadowRoots);
+          }
+          expandShadowRoots(document.body);
+        });
+
         const html = await page.content();
         const localFile = pageMap.get(cleanCurrentUrl) || 'index.html';
         fs.writeFileSync(path.join(siteDir, localFile), html);
-        await page.close();
+        await context.close();
       }
 
       const info = { url, timestamp: new Date().toISOString(), pages: pageCount, directory: siteDir };
       fs.writeFileSync(path.join(siteDir, 'clone-info.json'), JSON.stringify(info, null, 2));
+      
+      // Save Mock DB
+      fs.writeFileSync(path.join(siteDir, 'mock-db.json'), JSON.stringify(apiMocks, null, 2));
+      
+      // Create Service Worker
+      const swCode = `
+const MOCK_DB_URL = '/mock-db.json';
+let mockDb = null;
+
+self.addEventListener('install', (event) => {
+  self.skipWaiting();
+});
+
+self.addEventListener('activate', (event) => {
+  event.waitUntil(clients.claim());
+  event.waitUntil(
+    fetch(MOCK_DB_URL).then(res => res.json()).then(data => { mockDb = data; })
+  );
+});
+
+self.addEventListener('fetch', (event) => {
+  if (event.request.method === 'GET' && !event.request.url.includes('/mock-db.json') && mockDb) {
+    const mock = mockDb.find(m => m.url === event.request.url || event.request.url.includes(m.url));
+    if (mock) {
+      event.respondWith(
+        new Response(mock.body, {
+          status: mock.status,
+          headers: { 'Content-Type': mock.headers['content-type'] || 'application/json' }
+        })
+      );
+      return;
+    }
+  }
+});
+`;
+      fs.writeFileSync(path.join(siteDir, 'mock-sw.js'), swCode);
 
       return { success: true, path: siteDir, pagesCloned: pageCount };
     } catch (e) {