PRIMEIRO ENVIO

simple-crawler/test-crawler.js

@@ -0,0 +1,786 @@
+const express = require('express');
+const puppeteer = require('puppeteer');
+const cheerio = require('cheerio');
+const fs = require('fs');
+const path = require('path');
+const axios = require('axios');
+const { URL } = require('url');
+
+const app = express();
+app.use(express.json());
+
+class AdvancedWebCloner {
+  constructor() {
+    this.baseOutputDir = path.join(__dirname, '..', 'cloned-sites');
+    this.ensureDirectoryExists(this.baseOutputDir);
+    this.visitedPages = new Set();
+    this.downloadedAssets = new Map();
+    this.pageQueue = [];
+  }
+
+  ensureDirectoryExists(dirPath) {
+    if (!fs.existsSync(dirPath)) {
+      fs.mkdirSync(dirPath, { recursive: true });
+    }
+  }
+
+  sanitizeFilename(filename) {
+    return filename.replace(/[<>:"/\\|?*]/g, '_').replace(/\s+/g, '_').substring(0, 200);
+  }
+
+  getUniqueFilename(urlPath, existingFiles = new Set()) {
+    const parts = urlPath.split('/').filter(p => p);
+    const filename = parts.pop() || 'file';
+    const prefix = parts.slice(-3).join('_');
+    let baseName = this.sanitizeFilename(prefix ? `${prefix}_${filename}` : filename);
+    
+    // Garantir unicidade
+    let finalName = baseName;
+    let counter = 1;
+    while (existingFiles.has(finalName)) {
+      const ext = path.extname(baseName);
+      const name = baseName.replace(ext, '');
+      finalName = `${name}_${counter}${ext}`;
+      counter++;
+    }
+    
+    return finalName;
+  }
+
+  async downloadAsset(url, outputPath, retries = 3) {
+    for (let attempt = 1; attempt <= retries; attempt++) {
+      try {
+        // Para Google Fonts CSS, precisamos de user-agent específico para obter woff2
+        const isGoogleFonts = url.includes('fonts.googleapis.com');
+        
+        const response = await axios.get(url, {
+          responseType: 'arraybuffer',
+          timeout: 30000,
+          maxContentLength: 50 * 1024 * 1024, // 50MB
+          headers: {
+            'User-Agent': isGoogleFonts 
+              ? 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+              : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+            'Accept': '*/*',
+            'Accept-Encoding': 'gzip, deflate, br',
+            'Connection': 'keep-alive',
+            'Referer': isGoogleFonts ? 'https://fonts.googleapis.com/' : url
+          }
+        });
+        
+        this.ensureDirectoryExists(path.dirname(outputPath));
+        fs.writeFileSync(outputPath, response.data);
+        return true;
+      } catch (error) {
+        if (attempt === retries) {
+          // Não logar erro para fontes - muito verboso
+          if (!url.includes('.woff') && !url.includes('.ttf') && !url.includes('.eot')) {
+            console.log(`    ⚠️ Falha após ${retries} tentativas: ${url.substring(0, 80)}`);
+          }
+          return false;
+        }
+        await new Promise(resolve => setTimeout(resolve, 1000 * attempt));
+      }
+    }
+    return false;
+  }
+
+  getAssetInfo(assetUrl, baseUrl, siteDir, existingFiles) {
+    try {
+      const fullUrl = new URL(assetUrl, baseUrl);
+      const pathname = fullUrl.pathname;
+      
+      // Preservar a estrutura de pastas original
+      // Remove a barra inicial e mantém o caminho completo
+      let relativePath = pathname.startsWith('/') ? pathname.substring(1) : pathname;
+      
+      // Se o caminho estiver vazio, gerar um nome
+      if (!relativePath || relativePath === '') {
+        relativePath = 'index_' + Date.now();
+      }
+      
+      // Sanitizar apenas caracteres inválidos, mas manter a estrutura de pastas
+      relativePath = relativePath.replace(/[<>:"|?*]/g, '_');
+      
+      const fullPath = path.join(siteDir, relativePath);
+      
+      // Criar diretórios necessários
+      this.ensureDirectoryExists(path.dirname(fullPath));
+      
+      return {
+        fullPath: fullPath,
+        relativePath: relativePath,
+        fullUrl: fullUrl.href
+      };
+    } catch (error) {
+      return null;
+    }
+  }
+
+  extractUrlsFromCSS(cssContent, cssUrl) {
+    const urls = [];
+    
+    // Regex mais abrangente para capturar URLs em CSS
+    const patterns = [
+      /url\(['"]?([^'")\s]+)['"]?\)/g,  // url(...)
+      /@import\s+['"]([^'"]+)['"]/g,     // @import "..."
+      /@import\s+url\(['"]?([^'")\s]+)['"]?\)/g  // @import url(...)
+    ];
+    
+    patterns.forEach(regex => {
+      let match;
+      while ((match = regex.exec(cssContent)) !== null) {
+        const url = match[1];
+        if (url && !url.startsWith('data:') && !url.startsWith('#')) {
+          try {
+            const fullUrl = new URL(url, cssUrl).href;
+            urls.push({ original: url, full: fullUrl });
+          } catch(e) {
+            // Tentar como caminho relativo
+            try {
+              const baseUrl = new URL(cssUrl);
+              const resolvedUrl = new URL(url, baseUrl.origin + baseUrl.pathname.substring(0, baseUrl.pathname.lastIndexOf('/') + 1)).href;
+              urls.push({ original: url, full: resolvedUrl });
+            } catch(e2) {}
+          }
+        }
+      }
+    });
+    
+    return urls;
+  }
+
+  async extractAllAssets(page, pageUrl, $) {
+    const assets = new Map();
+    
+    // 1. CSS - todos os tipos
+    $('link[rel="stylesheet"], link[rel="preload"][as="style"]').each((_, el) => {
+      const href = $(el).attr('href');
+      if (href && !href.startsWith('data:')) {
+        try { assets.set(href, new URL(href, pageUrl).href); } catch(e) {}
+      }
+    });
+    
+    // 2. JavaScript - todos os tipos
+    $('script[src]').each((_, el) => {
+      const src = $(el).attr('src');
+      if (src && !src.startsWith('data:')) {
+        try { assets.set(src, new URL(src, pageUrl).href); } catch(e) {}
+      }
+    });
+    
+    // 3. Imagens - múltiplos atributos
+    $('img, picture source, [data-src], [data-lazy-src], [data-original]').each((_, el) => {
+      const attrs = ['src', 'data-src', 'data-lazy-src', 'data-original', 'data-srcset', 'srcset'];
+      attrs.forEach(attr => {
+        const value = $(el).attr(attr);
+        if (value && !value.startsWith('data:')) {
+          // Processar srcset
+          if (attr.includes('srcset')) {
+            value.split(',').forEach(item => {
+              const url = item.trim().split(' ')[0];
+              if (url && !url.startsWith('data:')) {
+                try { assets.set(url, new URL(url, pageUrl).href); } catch(e) {}
+              }
+            });
+          } else {
+            try { assets.set(value, new URL(value, pageUrl).href); } catch(e) {}
+          }
+        }
+      });
+    });
+    
+    // 4. Background images - style attributes e data attributes
+    $('[style*="url"], [data-background], [data-bg], [data-background-image]').each((_, el) => {
+      const attrs = ['style', 'data-background', 'data-bg', 'data-background-image'];
+      attrs.forEach(attr => {
+        const value = $(el).attr(attr) || '';
+        const matches = value.match(/url\(['"]?([^'")\s]+)['"]?\)/g);
+        if (matches) {
+          matches.forEach(match => {
+            const url = match.replace(/url\(['"]?/, '').replace(/['"]?\)/, '');
+            if (url && !url.startsWith('data:')) {
+              try { assets.set(url, new URL(url, pageUrl).href); } catch(e) {}
+            }
+          });
+        }
+      });
+    });
+    
+    // 5. Vídeos e áudio
+    $('video, audio, source, video[poster]').each((_, el) => {
+      const attrs = ['src', 'poster'];
+      attrs.forEach(attr => {
+        const value = $(el).attr(attr);
+        if (value && !value.startsWith('data:')) {
+          try { assets.set(value, new URL(value, pageUrl).href); } catch(e) {}
+        }
+      });
+    });
+    
+    // 6. Favicon e icons
+    $('link[rel*="icon"], link[rel="apple-touch-icon"]').each((_, el) => {
+      const href = $(el).attr('href');
+      if (href && !href.startsWith('data:')) {
+        try { assets.set(href, new URL(href, pageUrl).href); } catch(e) {}
+      }
+    });
+    
+    // 7. Preload e prefetch
+    $('link[rel="preload"], link[rel="prefetch"]').each((_, el) => {
+      const href = $(el).attr('href');
+      if (href && !href.startsWith('data:')) {
+        try { assets.set(href, new URL(href, pageUrl).href); } catch(e) {}
+      }
+    });
+    
+    // 8. Fonts do Google Fonts - baixar o CSS e as fontes
+    const googleFontsLinks = [];
+    $('link[href*="fonts.googleapis.com"]').each((_, el) => {
+      const href = $(el).attr('href');
+      if (href) {
+        googleFontsLinks.push(href);
+        try { assets.set(href, new URL(href, pageUrl).href); } catch(e) {}
+      }
+    });
+    
+    // 9. Fonts locais e de outros CDNs
+    $('link[href*="fonts.gstatic.com"], link[href*="font"]').each((_, el) => {
+      const href = $(el).attr('href');
+      if (href) {
+        try { assets.set(href, new URL(href, pageUrl).href); } catch(e) {}
+      }
+    });
+    
+    // 9. Extrair assets de inline styles
+    const inlineStyles = await page.evaluate(() => {
+      const styles = [];
+      document.querySelectorAll('style').forEach(style => {
+        styles.push(style.textContent);
+      });
+      return styles;
+    });
+    
+    inlineStyles.forEach(styleContent => {
+      const matches = styleContent.match(/url\(['"]?([^'")\s]+)['"]?\)/g);
+      if (matches) {
+        matches.forEach(match => {
+          const url = match.replace(/url\(['"]?/, '').replace(/['"]?\)/, '');
+          if (url && !url.startsWith('data:') && !url.startsWith('#')) {
+            try { assets.set(url, new URL(url, pageUrl).href); } catch(e) {}
+          }
+        });
+      }
+    });
+    
+    return assets;
+  }
+
+  async discoverInternalLinks(page, baseHostClean, startUrl) {
+    return await page.evaluate((baseHostClean, startUrl) => {
+      const links = new Set();
+      const baseUrl = new URL(startUrl);
+      
+      document.querySelectorAll('a[href]').forEach(a => {
+        try {
+          const href = a.href;
+          const url = new URL(href);
+          
+          // Verificar se é link interno
+          if (url.hostname.includes(baseHostClean) || url.hostname === baseUrl.hostname) {
+            // Filtrar extensões de arquivo
+            if (!url.pathname.match(/\.(css|js|png|jpg|jpeg|gif|svg|webp|ico|pdf|zip|doc|docx|xls|xlsx|mp4|mp3|avi)$/i)) {
+              // Remover query strings e fragments para normalizar
+              const cleanUrl = `${url.origin}${url.pathname}`.replace(/\/$/, '');
+              if (cleanUrl && 
+                  !cleanUrl.includes('mailto:') && 
+                  !cleanUrl.includes('tel:') &&
+                  !cleanUrl.includes('javascript:')) {
+                links.add(cleanUrl);
+              }
+            }
+          }
+        } catch(e) {}
+      });
+      
+      return Array.from(links);
+    }, baseHostClean, startUrl);
+  }
+
+  async cloneWebsite(startUrl) {
+    const browser = await puppeteer.launch({
+      headless: 'new',
+      args: [
+        '--no-sandbox',
+        '--disable-setuid-sandbox',
+        '--disable-dev-shm-usage',
+        '--disable-accelerated-2d-canvas',
+        '--disable-gpu'
+      ]
+    });
+    
+    try {
+      const page = await browser.newPage();
+      
+      // Configurar viewport grande para capturar mais conteúdo
+      await page.setViewport({ width: 1920, height: 1080 });
+      
+      // Configurar user agent realista
+      await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
+      
+      console.log(`\n🚀 INICIANDO CLONAGEM PROFUNDA DE: ${startUrl}\n`);
+      
+      const parsedUrl = new URL(startUrl);
+      const baseHost = parsedUrl.hostname;
+      const baseHostClean = baseHost.replace('www.', '');
+      
+      // Criar diretório
+      const siteName = this.sanitizeFilename(baseHost);
+      const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+      const siteDir = path.join(this.baseOutputDir, `${siteName}_${timestamp}`);
+      this.ensureDirectoryExists(siteDir);
+      
+      console.log(`📁 Diretório: ${siteDir}\n`);
+      
+      // Fase 1: Descobrir todas as páginas
+      console.log(`🔍 FASE 1: Descobrindo páginas do site...`);
+      
+      const response = await page.goto(startUrl, {
+        waitUntil: 'networkidle0',
+        timeout: 60000
+      });
+      
+      await page.waitForTimeout(3000);
+      
+      // Descobrir links internos
+      const internalLinks = await this.discoverInternalLinks(page, baseHostClean, startUrl);
+      
+      // Criar mapa de páginas
+      const pageMap = new Map();
+      const existingFiles = new Set();
+      
+      // Adicionar página inicial
+      pageMap.set(startUrl, { file: 'index.html', title: 'Home' });
+      existingFiles.add('index.html');
+      
+      // Adicionar páginas descobertas
+      for (const link of internalLinks) {
+        if (!pageMap.has(link)) {
+          const urlObj = new URL(link);
+          let pathname = urlObj.pathname.replace(/^\/|\/$/g, '');
+          
+          let filename;
+          if (!pathname) {
+            filename = 'home.html';
+          } else {
+            filename = pathname.replace(/\//g, '_') + '.html';
+          }
+          
+          // Garantir unicidade
+          let finalFilename = filename;
+          let counter = 1;
+          while (existingFiles.has(finalFilename)) {
+            finalFilename = filename.replace('.html', `_${counter}.html`);
+            counter++;
+          }
+          
+          pageMap.set(link, { file: finalFilename, title: pathname || 'page' });
+          existingFiles.add(finalFilename);
+        }
+      }
+      
+      console.log(`✅ Encontradas ${pageMap.size} páginas para clonar\n`);
+      
+      // Fase 2: Clonar cada página
+      console.log(`📄 FASE 2: Clonando páginas e coletando assets...`);
+      
+      const globalAssets = new Map();
+      const pageContents = new Map();
+      const assetFiles = new Set();
+      
+      let pageIndex = 0;
+      for (const [pageUrl, pageInfo] of pageMap) {
+        pageIndex++;
+        console.log(`\n  [${pageIndex}/${pageMap.size}] ${pageInfo.file}`);
+        
+        try {
+          await page.goto(pageUrl, {
+            waitUntil: 'networkidle0',
+            timeout: 60000
+          });
+          
+          // Esperar carregamento completo
+          await page.waitForTimeout(3000);
+          
+          // Scroll completo para carregar lazy content
+          console.log(`    📜 Fazendo scroll para carregar conteúdo lazy...`);
+          await page.evaluate(async () => {
+            await new Promise((resolve) => {
+              let totalHeight = 0;
+              const distance = 300;
+              const timer = setInterval(() => {
+                const scrollHeight = document.body.scrollHeight;
+                window.scrollBy(0, distance);
+                totalHeight += distance;
+                
+                if (totalHeight >= scrollHeight) {
+                  clearInterval(timer);
+                  window.scrollTo(0, 0);
+                  setTimeout(resolve, 1000);
+                }
+              }, 100);
+            });
+          });
+          
+          await page.waitForTimeout(2000);
+          
+          // Capturar conteúdo
+          const content = await page.content();
+          pageContents.set(pageUrl, content);
+          
+          const $ = cheerio.load(content);
+          
+          // Extrair todos os assets
+          console.log(`    🔍 Extraindo assets...`);
+          const pageAssets = await this.extractAllAssets(page, pageUrl, $);
+          
+          // Adicionar ao conjunto global
+          for (const [original, full] of pageAssets) {
+            globalAssets.set(original, full);
+          }
+          
+          console.log(`    ✓ ${pageAssets.size} assets encontrados`);
+          
+        } catch (err) {
+          console.log(`    ❌ Erro: ${err.message}`);
+        }
+      }
+      
+      console.log(`\n✅ Total de ${globalAssets.size} assets únicos encontrados\n`);
+      
+      // Fase 3: Baixar todos os assets
+      console.log(`📥 FASE 3: Baixando assets...`);
+      
+      const replacements = new Map();
+      const batchSize = 5; // Reduzido para evitar sobrecarga
+      const assetEntries = [...globalAssets.entries()];
+      let successCount = 0;
+      let failCount = 0;
+      
+      for (let i = 0; i < assetEntries.length; i += batchSize) {
+        const batch = assetEntries.slice(i, i + batchSize);
+        const promises = batch.map(async ([originalUrl, fullUrl]) => {
+          const info = this.getAssetInfo(fullUrl, startUrl, siteDir, assetFiles);
+          if (info) {
+            const ok = await this.downloadAsset(info.fullUrl, info.fullPath);
+            if (ok) {
+              replacements.set(originalUrl, info.relativePath);
+              replacements.set(info.fullUrl, info.relativePath);
+              
+              // Variações de protocolo
+              if (info.fullUrl.startsWith('https://')) {
+                replacements.set(info.fullUrl.replace('https://', 'http://'), info.relativePath);
+              } else if (info.fullUrl.startsWith('http://')) {
+                replacements.set(info.fullUrl.replace('http://', 'https://'), info.relativePath);
+              }
+              
+              successCount++;
+              return { ok: true, info, fullUrl };
+            } else {
+              failCount++;
+            }
+          }
+          return { ok: false };
+        });
+        
+        await Promise.all(promises);
+        
+        const progress = Math.round((i + batch.length) / assetEntries.length * 100);
+        console.log(`  📊 Progresso: ${progress}% (${successCount} ok, ${failCount} falhas)`);
+      }
+      
+      console.log(`\n✅ ${successCount} assets baixados com sucesso\n`);
+      
+      // Fase 4: Extrair assets de CSS
+      console.log(`🎨 FASE 4: Extraindo assets de arquivos CSS...`);
+      
+      const cssAssets = new Map();
+      let cssAssetCount = 0;
+      
+      for (const [original, local] of replacements) {
+        if (local.endsWith('.css')) {
+          try {
+            const cssPath = path.join(siteDir, local);
+            if (fs.existsSync(cssPath)) {
+              const cssContent = fs.readFileSync(cssPath, 'utf8');
+              
+              // Determinar URL base do CSS
+              let cssBaseUrl = original;
+              for (const [origUrl, localPath] of replacements) {
+                if (localPath === local) {
+                  cssBaseUrl = origUrl;
+                  break;
+                }
+              }
+              
+              const cssUrls = this.extractUrlsFromCSS(cssContent, cssBaseUrl);
+              
+              for (const cssUrl of cssUrls) {
+                if (!replacements.has(cssUrl.full) && !cssAssets.has(cssUrl.full)) {
+                  cssAssets.set(cssUrl.original, cssUrl.full);
+                }
+              }
+            }
+          } catch(e) {}
+        }
+      }
+      
+      if (cssAssets.size > 0) {
+        console.log(`  📦 Encontrados ${cssAssets.size} assets em CSS (fontes, imagens, etc.)`);
+        
+        for (const [originalUrl, fullUrl] of cssAssets) {
+          const info = this.getAssetInfo(fullUrl, startUrl, siteDir, assetFiles);
+          if (info) {
+            const ok = await this.downloadAsset(info.fullUrl, info.fullPath);
+            if (ok) {
+              replacements.set(originalUrl, info.relativePath);
+              replacements.set(info.fullUrl, info.relativePath);
+              cssAssetCount++;
+              
+              // Atualizar CSS files para substituir URLs
+              for (const [origCss, localCss] of replacements) {
+                if (localCss.endsWith('.css')) {
+                  try {
+                    const cssPath = path.join(siteDir, localCss);
+                    if (fs.existsSync(cssPath)) {
+                      let cssContent = fs.readFileSync(cssPath, 'utf8');
+                      const escaped = originalUrl.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+                      cssContent = cssContent.replace(new RegExp(escaped, 'g'), info.relativePath);
+                      fs.writeFileSync(cssPath, cssContent, 'utf8');
+                    }
+                  } catch(e) {}
+                }
+              }
+            }
+          }
+        }
+        
+        console.log(`  ✅ ${cssAssetCount} assets de CSS baixados`);
+      } else {
+        console.log(`  ℹ️ Nenhum asset adicional encontrado em CSS`);
+      }
+      
+      // Fase 5: Processar e salvar páginas
+      console.log(`\n💾 FASE 5: Processando e salvando páginas HTML...\n`);
+      
+      const sortedReplacements = [...replacements.entries()]
+        .sort((a, b) => b[0].length - a[0].length);
+      
+      const elementsToRemove = [
+        '.cookie-banner', '.cookie-notice', '.cookie-consent', '.cookies-banner',
+        '.lgpd-banner', '.lgpd-notice', '.lgpd-popup', '.lgpd',
+        '#cookie-banner', '#cookie-notice', '#cookie-consent', '#cookies',
+        '#lgpd-banner', '#lgpd', '#lgpd-notice',
+        '[class*="cookie"]', '[class*="lgpd"]', '[class*="gdpr"]',
+        '[id*="cookie"]', '[id*="lgpd"]', '[id*="gdpr"]',
+        '.popup-overlay', '.modal-overlay', '.consent-banner',
+        '[class*="consent"]', '[class*="privacy-banner"]',
+        '.joinchat', '#joinchat', '.joinchat__button',
+      ];
+      
+      let pagesSaved = 0;
+      for (const [pageUrl, pageInfo] of pageMap) {
+        let html = pageContents.get(pageUrl);
+        if (!html) continue;
+        
+        console.log(`  💾 ${pageInfo.file}`);
+        
+        // Remover elementos indesejados
+        const $ = cheerio.load(html);
+        
+        for (const selector of elementsToRemove) {
+          try { $(selector).remove(); } catch(e) {}
+        }
+        
+        // Remover scripts problemáticos e externos que não funcionam offline
+        $('script').each((_, el) => {
+          const src = $(el).attr('src') || '';
+          const content = $(el).html() || '';
+          
+          // Lista de scripts a remover
+          const removePatterns = [
+            'cookie', 'lgpd', 'consent', 'gdpr',
+            'recaptcha', 'google-analytics', 'googletagmanager', 'gtag', 'gtm',
+            'facebook.net', 'fbevents', 'fbq',
+            'doubleclick.net', 'googleadservices',
+            'analytics.js', 'ga.js'
+          ];
+          
+          const shouldRemove = removePatterns.some(pattern => 
+            src.includes(pattern) || content.includes(pattern)
+          );
+          
+          if (shouldRemove && !src.includes('jquery') && !src.includes('elementor')) {
+            $(el).remove();
+          }
+        });
+        
+        // Remover iframes externos (Google Tag Manager, etc)
+        $('iframe[src*="googletagmanager"], iframe[src*="facebook"], iframe[src*="doubleclick"]').remove();
+        
+        // Remover noscript tags de tracking
+        $('noscript').each((_, el) => {
+          const content = $(el).html() || '';
+          if (content.includes('googletagmanager') || content.includes('facebook')) {
+            $(el).remove();
+          }
+        });
+        
+        html = $.html();
+        
+        // Substituir assets - usar substituição mais precisa
+        for (const [original, local] of sortedReplacements) {
+          // Escapar caracteres especiais
+          const escaped = original.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+          
+          // Substituir em diferentes contextos
+          // 1. Em atributos HTML (src, href, data-src, etc)
+          html = html.replace(new RegExp(`(src|href|data-src|data-lazy-src|data-background|poster)=["']${escaped}["']`, 'gi'), `$1="${local}"`);
+          
+          // 2. Em CSS inline (background-image, etc)
+          html = html.replace(new RegExp(`url\\(['"]?${escaped}['"]?\\)`, 'gi'), `url("${local}")`);
+          
+          // 3. Substituição direta
+          html = html.replace(new RegExp(escaped, 'g'), local);
+        }
+        
+        // Substituir links internos
+        for (const [url, info] of pageMap) {
+          const escaped = url.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+          html = html.replace(new RegExp(`href=["']${escaped}["']`, 'gi'), `href="${info.file}"`);
+          
+          const urlObj = new URL(url);
+          if (urlObj.pathname && urlObj.pathname !== '/') {
+            const pathname = urlObj.pathname.replace(/\/$/, '');
+            html = html.replace(
+              new RegExp(`href=["']${pathname.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}["']`, 'gi'),
+              `href="${info.file}"`
+            );
+          }
+        }
+        
+        // Links restantes do domínio -> index.html
+        html = html.replace(
+          new RegExp(`href=["']https?://(www\\.)?${baseHostClean.replace(/\./g, '\\.')}[^"']*["']`, 'gi'),
+          'href="index.html"'
+        );
+        
+        // Adicionar script para suprimir erros e melhorar compatibilidade offline
+        const offlineScript = `
+<script>
+// Suprimir erros de scripts externos
+window.addEventListener('error', function(e) {
+  // Ignorar erros de recursos externos
+  if (e.filename && (
+    e.filename.includes('recaptcha') ||
+    e.filename.includes('google') ||
+    e.filename.includes('facebook') ||
+    e.filename.includes('doubleclick')
+  )) {
+    e.preventDefault();
+    return true;
+  }
+}, true);
+
+// Criar stubs para funções que podem estar faltando
+window.fbq = window.fbq || function() { console.log('fbq stub called'); };
+window.gtag = window.gtag || function() { console.log('gtag stub called'); };
+window.dataLayer = window.dataLayer || [];
+window.ga = window.ga || function() { console.log('ga stub called'); };
+
+// Desabilitar reCAPTCHA se presente
+if (typeof grecaptcha !== 'undefined') {
+  grecaptcha.ready = function(cb) { console.log('reCAPTCHA disabled offline'); };
+}
+
+console.log('🌐 Site clonado carregado em modo offline');
+</script>
+`;
+        
+        // Inserir antes do </body>
+        html = html.replace('</body>', offlineScript + '</body>');
+        
+        fs.writeFileSync(path.join(siteDir, pageInfo.file), html, 'utf8');
+        pagesSaved++;
+      }
+      
+      // Salvar informações
+      const info = {
+        originalUrl: startUrl,
+        clonedAt: new Date().toISOString(),
+        pages: [...pageMap.entries()].map(([url, info]) => ({ url, file: info.file })),
+        assetsDownloaded: successCount + cssAssetCount,
+        assetsFailed: failCount,
+        totalAssets: globalAssets.size + cssAssets.size,
+        directory: siteDir
+      };
+      
+      fs.writeFileSync(path.join(siteDir, 'clone-info.json'), JSON.stringify(info, null, 2));
+      fs.writeFileSync(path.join(siteDir, 'abrir-site.bat'), `@echo off\nstart index.html`);
+      
+      console.log(`\n${'='.repeat(60)}`);
+      console.log(`🎉 CLONAGEM CONCLUÍDA COM SUCESSO!`);
+      console.log(`${'='.repeat(60)}`);
+      console.log(`📄 Páginas clonadas: ${pagesSaved}`);
+      console.log(`📦 Assets baixados: ${successCount + cssAssetCount}`);
+      console.log(`⚠️  Assets com falha: ${failCount}`);
+      console.log(`📁 Localização: ${siteDir}`);
+      console.log(`${'='.repeat(60)}\n`);
+      
+      return {
+        url: startUrl,
+        title: 'Site Clonado',
+        statusCode: response.status(),
+        links: [],
+        images: [],
+        contentLength: 0,
+        timestamp: new Date().toISOString(),
+        cloneInfo: {
+          directory: siteDir,
+          pagesCloned: pagesSaved,
+          assetsDownloaded: successCount + cssAssetCount,
+          assetsFailed: failCount,
+          totalAssets: globalAssets.size + cssAssets.size,
+          success: true
+        }
+      };
+      
+    } finally {
+      await browser.close();
+    }
+  }
+}
+
+// API
+app.post('/crawl', async (req, res) => {
+  try {
+    const { url } = req.body;
+    if (!url) return res.status(400).json({ error: 'URL obrigatória' });
+    
+    const cloner = new AdvancedWebCloner();
+    const result = await cloner.cloneWebsite(url);
+    res.json({ success: true, data: result });
+  } catch (error) {
+    console.error('❌ Erro:', error.message);
+    res.status(500).json({ success: false, error: error.message });
+  }
+});
+
+app.get('/health', (req, res) => {
+  res.json({ status: 'ok', service: 'advanced-web-cloner' });
+});
+
+app.listen(3001, () => {
+  console.log('🌐 Advanced Web Cloner rodando em http://localhost:3001');
+  console.log('🚀 Pronto para clonagem profunda de sites!\n');
+});