787 lines
28 KiB
JavaScript
787 lines
28 KiB
JavaScript
const express = require('express');
|
||
const puppeteer = require('puppeteer');
|
||
const cheerio = require('cheerio');
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
const axios = require('axios');
|
||
const { URL } = require('url');
|
||
|
||
const app = express();
|
||
app.use(express.json());
|
||
|
||
class AdvancedWebCloner {
|
||
constructor() {
|
||
this.baseOutputDir = path.join(__dirname, '..', 'cloned-sites');
|
||
this.ensureDirectoryExists(this.baseOutputDir);
|
||
this.visitedPages = new Set();
|
||
this.downloadedAssets = new Map();
|
||
this.pageQueue = [];
|
||
}
|
||
|
||
ensureDirectoryExists(dirPath) {
|
||
if (!fs.existsSync(dirPath)) {
|
||
fs.mkdirSync(dirPath, { recursive: true });
|
||
}
|
||
}
|
||
|
||
sanitizeFilename(filename) {
|
||
return filename.replace(/[<>:"/\\|?*]/g, '_').replace(/\s+/g, '_').substring(0, 200);
|
||
}
|
||
|
||
getUniqueFilename(urlPath, existingFiles = new Set()) {
|
||
const parts = urlPath.split('/').filter(p => p);
|
||
const filename = parts.pop() || 'file';
|
||
const prefix = parts.slice(-3).join('_');
|
||
let baseName = this.sanitizeFilename(prefix ? `${prefix}_${filename}` : filename);
|
||
|
||
// Garantir unicidade
|
||
let finalName = baseName;
|
||
let counter = 1;
|
||
while (existingFiles.has(finalName)) {
|
||
const ext = path.extname(baseName);
|
||
const name = baseName.replace(ext, '');
|
||
finalName = `${name}_${counter}${ext}`;
|
||
counter++;
|
||
}
|
||
|
||
return finalName;
|
||
}
|
||
|
||
async downloadAsset(url, outputPath, retries = 3) {
|
||
for (let attempt = 1; attempt <= retries; attempt++) {
|
||
try {
|
||
// Para Google Fonts CSS, precisamos de user-agent específico para obter woff2
|
||
const isGoogleFonts = url.includes('fonts.googleapis.com');
|
||
|
||
const response = await axios.get(url, {
|
||
responseType: 'arraybuffer',
|
||
timeout: 30000,
|
||
maxContentLength: 50 * 1024 * 1024, // 50MB
|
||
headers: {
|
||
'User-Agent': isGoogleFonts
|
||
? 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
||
: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||
'Accept': '*/*',
|
||
'Accept-Encoding': 'gzip, deflate, br',
|
||
'Connection': 'keep-alive',
|
||
'Referer': isGoogleFonts ? 'https://fonts.googleapis.com/' : url
|
||
}
|
||
});
|
||
|
||
this.ensureDirectoryExists(path.dirname(outputPath));
|
||
fs.writeFileSync(outputPath, response.data);
|
||
return true;
|
||
} catch (error) {
|
||
if (attempt === retries) {
|
||
// Não logar erro para fontes - muito verboso
|
||
if (!url.includes('.woff') && !url.includes('.ttf') && !url.includes('.eot')) {
|
||
console.log(` ⚠️ Falha após ${retries} tentativas: ${url.substring(0, 80)}`);
|
||
}
|
||
return false;
|
||
}
|
||
await new Promise(resolve => setTimeout(resolve, 1000 * attempt));
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
|
||
getAssetInfo(assetUrl, baseUrl, siteDir, existingFiles) {
|
||
try {
|
||
const fullUrl = new URL(assetUrl, baseUrl);
|
||
const pathname = fullUrl.pathname;
|
||
|
||
// Preservar a estrutura de pastas original
|
||
// Remove a barra inicial e mantém o caminho completo
|
||
let relativePath = pathname.startsWith('/') ? pathname.substring(1) : pathname;
|
||
|
||
// Se o caminho estiver vazio, gerar um nome
|
||
if (!relativePath || relativePath === '') {
|
||
relativePath = 'index_' + Date.now();
|
||
}
|
||
|
||
// Sanitizar apenas caracteres inválidos, mas manter a estrutura de pastas
|
||
relativePath = relativePath.replace(/[<>:"|?*]/g, '_');
|
||
|
||
const fullPath = path.join(siteDir, relativePath);
|
||
|
||
// Criar diretórios necessários
|
||
this.ensureDirectoryExists(path.dirname(fullPath));
|
||
|
||
return {
|
||
fullPath: fullPath,
|
||
relativePath: relativePath,
|
||
fullUrl: fullUrl.href
|
||
};
|
||
} catch (error) {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
extractUrlsFromCSS(cssContent, cssUrl) {
|
||
const urls = [];
|
||
|
||
// Regex mais abrangente para capturar URLs em CSS
|
||
const patterns = [
|
||
/url\(['"]?([^'")\s]+)['"]?\)/g, // url(...)
|
||
/@import\s+['"]([^'"]+)['"]/g, // @import "..."
|
||
/@import\s+url\(['"]?([^'")\s]+)['"]?\)/g // @import url(...)
|
||
];
|
||
|
||
patterns.forEach(regex => {
|
||
let match;
|
||
while ((match = regex.exec(cssContent)) !== null) {
|
||
const url = match[1];
|
||
if (url && !url.startsWith('data:') && !url.startsWith('#')) {
|
||
try {
|
||
const fullUrl = new URL(url, cssUrl).href;
|
||
urls.push({ original: url, full: fullUrl });
|
||
} catch(e) {
|
||
// Tentar como caminho relativo
|
||
try {
|
||
const baseUrl = new URL(cssUrl);
|
||
const resolvedUrl = new URL(url, baseUrl.origin + baseUrl.pathname.substring(0, baseUrl.pathname.lastIndexOf('/') + 1)).href;
|
||
urls.push({ original: url, full: resolvedUrl });
|
||
} catch(e2) {}
|
||
}
|
||
}
|
||
}
|
||
});
|
||
|
||
return urls;
|
||
}
|
||
|
||
async extractAllAssets(page, pageUrl, $) {
|
||
const assets = new Map();
|
||
|
||
// 1. CSS - todos os tipos
|
||
$('link[rel="stylesheet"], link[rel="preload"][as="style"]').each((_, el) => {
|
||
const href = $(el).attr('href');
|
||
if (href && !href.startsWith('data:')) {
|
||
try { assets.set(href, new URL(href, pageUrl).href); } catch(e) {}
|
||
}
|
||
});
|
||
|
||
// 2. JavaScript - todos os tipos
|
||
$('script[src]').each((_, el) => {
|
||
const src = $(el).attr('src');
|
||
if (src && !src.startsWith('data:')) {
|
||
try { assets.set(src, new URL(src, pageUrl).href); } catch(e) {}
|
||
}
|
||
});
|
||
|
||
// 3. Imagens - múltiplos atributos
|
||
$('img, picture source, [data-src], [data-lazy-src], [data-original]').each((_, el) => {
|
||
const attrs = ['src', 'data-src', 'data-lazy-src', 'data-original', 'data-srcset', 'srcset'];
|
||
attrs.forEach(attr => {
|
||
const value = $(el).attr(attr);
|
||
if (value && !value.startsWith('data:')) {
|
||
// Processar srcset
|
||
if (attr.includes('srcset')) {
|
||
value.split(',').forEach(item => {
|
||
const url = item.trim().split(' ')[0];
|
||
if (url && !url.startsWith('data:')) {
|
||
try { assets.set(url, new URL(url, pageUrl).href); } catch(e) {}
|
||
}
|
||
});
|
||
} else {
|
||
try { assets.set(value, new URL(value, pageUrl).href); } catch(e) {}
|
||
}
|
||
}
|
||
});
|
||
});
|
||
|
||
// 4. Background images - style attributes e data attributes
|
||
$('[style*="url"], [data-background], [data-bg], [data-background-image]').each((_, el) => {
|
||
const attrs = ['style', 'data-background', 'data-bg', 'data-background-image'];
|
||
attrs.forEach(attr => {
|
||
const value = $(el).attr(attr) || '';
|
||
const matches = value.match(/url\(['"]?([^'")\s]+)['"]?\)/g);
|
||
if (matches) {
|
||
matches.forEach(match => {
|
||
const url = match.replace(/url\(['"]?/, '').replace(/['"]?\)/, '');
|
||
if (url && !url.startsWith('data:')) {
|
||
try { assets.set(url, new URL(url, pageUrl).href); } catch(e) {}
|
||
}
|
||
});
|
||
}
|
||
});
|
||
});
|
||
|
||
// 5. Vídeos e áudio
|
||
$('video, audio, source, video[poster]').each((_, el) => {
|
||
const attrs = ['src', 'poster'];
|
||
attrs.forEach(attr => {
|
||
const value = $(el).attr(attr);
|
||
if (value && !value.startsWith('data:')) {
|
||
try { assets.set(value, new URL(value, pageUrl).href); } catch(e) {}
|
||
}
|
||
});
|
||
});
|
||
|
||
// 6. Favicon e icons
|
||
$('link[rel*="icon"], link[rel="apple-touch-icon"]').each((_, el) => {
|
||
const href = $(el).attr('href');
|
||
if (href && !href.startsWith('data:')) {
|
||
try { assets.set(href, new URL(href, pageUrl).href); } catch(e) {}
|
||
}
|
||
});
|
||
|
||
// 7. Preload e prefetch
|
||
$('link[rel="preload"], link[rel="prefetch"]').each((_, el) => {
|
||
const href = $(el).attr('href');
|
||
if (href && !href.startsWith('data:')) {
|
||
try { assets.set(href, new URL(href, pageUrl).href); } catch(e) {}
|
||
}
|
||
});
|
||
|
||
// 8. Fonts do Google Fonts - baixar o CSS e as fontes
|
||
const googleFontsLinks = [];
|
||
$('link[href*="fonts.googleapis.com"]').each((_, el) => {
|
||
const href = $(el).attr('href');
|
||
if (href) {
|
||
googleFontsLinks.push(href);
|
||
try { assets.set(href, new URL(href, pageUrl).href); } catch(e) {}
|
||
}
|
||
});
|
||
|
||
// 9. Fonts locais e de outros CDNs
|
||
$('link[href*="fonts.gstatic.com"], link[href*="font"]').each((_, el) => {
|
||
const href = $(el).attr('href');
|
||
if (href) {
|
||
try { assets.set(href, new URL(href, pageUrl).href); } catch(e) {}
|
||
}
|
||
});
|
||
|
||
// 9. Extrair assets de inline styles
|
||
const inlineStyles = await page.evaluate(() => {
|
||
const styles = [];
|
||
document.querySelectorAll('style').forEach(style => {
|
||
styles.push(style.textContent);
|
||
});
|
||
return styles;
|
||
});
|
||
|
||
inlineStyles.forEach(styleContent => {
|
||
const matches = styleContent.match(/url\(['"]?([^'")\s]+)['"]?\)/g);
|
||
if (matches) {
|
||
matches.forEach(match => {
|
||
const url = match.replace(/url\(['"]?/, '').replace(/['"]?\)/, '');
|
||
if (url && !url.startsWith('data:') && !url.startsWith('#')) {
|
||
try { assets.set(url, new URL(url, pageUrl).href); } catch(e) {}
|
||
}
|
||
});
|
||
}
|
||
});
|
||
|
||
return assets;
|
||
}
|
||
|
||
async discoverInternalLinks(page, baseHostClean, startUrl) {
|
||
return await page.evaluate((baseHostClean, startUrl) => {
|
||
const links = new Set();
|
||
const baseUrl = new URL(startUrl);
|
||
|
||
document.querySelectorAll('a[href]').forEach(a => {
|
||
try {
|
||
const href = a.href;
|
||
const url = new URL(href);
|
||
|
||
// Verificar se é link interno
|
||
if (url.hostname.includes(baseHostClean) || url.hostname === baseUrl.hostname) {
|
||
// Filtrar extensões de arquivo
|
||
if (!url.pathname.match(/\.(css|js|png|jpg|jpeg|gif|svg|webp|ico|pdf|zip|doc|docx|xls|xlsx|mp4|mp3|avi)$/i)) {
|
||
// Remover query strings e fragments para normalizar
|
||
const cleanUrl = `${url.origin}${url.pathname}`.replace(/\/$/, '');
|
||
if (cleanUrl &&
|
||
!cleanUrl.includes('mailto:') &&
|
||
!cleanUrl.includes('tel:') &&
|
||
!cleanUrl.includes('javascript:')) {
|
||
links.add(cleanUrl);
|
||
}
|
||
}
|
||
}
|
||
} catch(e) {}
|
||
});
|
||
|
||
return Array.from(links);
|
||
}, baseHostClean, startUrl);
|
||
}
|
||
|
||
async cloneWebsite(startUrl) {
|
||
const browser = await puppeteer.launch({
|
||
headless: 'new',
|
||
args: [
|
||
'--no-sandbox',
|
||
'--disable-setuid-sandbox',
|
||
'--disable-dev-shm-usage',
|
||
'--disable-accelerated-2d-canvas',
|
||
'--disable-gpu'
|
||
]
|
||
});
|
||
|
||
try {
|
||
const page = await browser.newPage();
|
||
|
||
// Configurar viewport grande para capturar mais conteúdo
|
||
await page.setViewport({ width: 1920, height: 1080 });
|
||
|
||
// Configurar user agent realista
|
||
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
|
||
|
||
console.log(`\n🚀 INICIANDO CLONAGEM PROFUNDA DE: ${startUrl}\n`);
|
||
|
||
const parsedUrl = new URL(startUrl);
|
||
const baseHost = parsedUrl.hostname;
|
||
const baseHostClean = baseHost.replace('www.', '');
|
||
|
||
// Criar diretório
|
||
const siteName = this.sanitizeFilename(baseHost);
|
||
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
||
const siteDir = path.join(this.baseOutputDir, `${siteName}_${timestamp}`);
|
||
this.ensureDirectoryExists(siteDir);
|
||
|
||
console.log(`📁 Diretório: ${siteDir}\n`);
|
||
|
||
// Fase 1: Descobrir todas as páginas
|
||
console.log(`🔍 FASE 1: Descobrindo páginas do site...`);
|
||
|
||
const response = await page.goto(startUrl, {
|
||
waitUntil: 'networkidle0',
|
||
timeout: 60000
|
||
});
|
||
|
||
await page.waitForTimeout(3000);
|
||
|
||
// Descobrir links internos
|
||
const internalLinks = await this.discoverInternalLinks(page, baseHostClean, startUrl);
|
||
|
||
// Criar mapa de páginas
|
||
const pageMap = new Map();
|
||
const existingFiles = new Set();
|
||
|
||
// Adicionar página inicial
|
||
pageMap.set(startUrl, { file: 'index.html', title: 'Home' });
|
||
existingFiles.add('index.html');
|
||
|
||
// Adicionar páginas descobertas
|
||
for (const link of internalLinks) {
|
||
if (!pageMap.has(link)) {
|
||
const urlObj = new URL(link);
|
||
let pathname = urlObj.pathname.replace(/^\/|\/$/g, '');
|
||
|
||
let filename;
|
||
if (!pathname) {
|
||
filename = 'home.html';
|
||
} else {
|
||
filename = pathname.replace(/\//g, '_') + '.html';
|
||
}
|
||
|
||
// Garantir unicidade
|
||
let finalFilename = filename;
|
||
let counter = 1;
|
||
while (existingFiles.has(finalFilename)) {
|
||
finalFilename = filename.replace('.html', `_${counter}.html`);
|
||
counter++;
|
||
}
|
||
|
||
pageMap.set(link, { file: finalFilename, title: pathname || 'page' });
|
||
existingFiles.add(finalFilename);
|
||
}
|
||
}
|
||
|
||
console.log(`✅ Encontradas ${pageMap.size} páginas para clonar\n`);
|
||
|
||
// Fase 2: Clonar cada página
|
||
console.log(`📄 FASE 2: Clonando páginas e coletando assets...`);
|
||
|
||
const globalAssets = new Map();
|
||
const pageContents = new Map();
|
||
const assetFiles = new Set();
|
||
|
||
let pageIndex = 0;
|
||
for (const [pageUrl, pageInfo] of pageMap) {
|
||
pageIndex++;
|
||
console.log(`\n [${pageIndex}/${pageMap.size}] ${pageInfo.file}`);
|
||
|
||
try {
|
||
await page.goto(pageUrl, {
|
||
waitUntil: 'networkidle0',
|
||
timeout: 60000
|
||
});
|
||
|
||
// Esperar carregamento completo
|
||
await page.waitForTimeout(3000);
|
||
|
||
// Scroll completo para carregar lazy content
|
||
console.log(` 📜 Fazendo scroll para carregar conteúdo lazy...`);
|
||
await page.evaluate(async () => {
|
||
await new Promise((resolve) => {
|
||
let totalHeight = 0;
|
||
const distance = 300;
|
||
const timer = setInterval(() => {
|
||
const scrollHeight = document.body.scrollHeight;
|
||
window.scrollBy(0, distance);
|
||
totalHeight += distance;
|
||
|
||
if (totalHeight >= scrollHeight) {
|
||
clearInterval(timer);
|
||
window.scrollTo(0, 0);
|
||
setTimeout(resolve, 1000);
|
||
}
|
||
}, 100);
|
||
});
|
||
});
|
||
|
||
await page.waitForTimeout(2000);
|
||
|
||
// Capturar conteúdo
|
||
const content = await page.content();
|
||
pageContents.set(pageUrl, content);
|
||
|
||
const $ = cheerio.load(content);
|
||
|
||
// Extrair todos os assets
|
||
console.log(` 🔍 Extraindo assets...`);
|
||
const pageAssets = await this.extractAllAssets(page, pageUrl, $);
|
||
|
||
// Adicionar ao conjunto global
|
||
for (const [original, full] of pageAssets) {
|
||
globalAssets.set(original, full);
|
||
}
|
||
|
||
console.log(` ✓ ${pageAssets.size} assets encontrados`);
|
||
|
||
} catch (err) {
|
||
console.log(` ❌ Erro: ${err.message}`);
|
||
}
|
||
}
|
||
|
||
console.log(`\n✅ Total de ${globalAssets.size} assets únicos encontrados\n`);
|
||
|
||
// Fase 3: Baixar todos os assets
|
||
console.log(`📥 FASE 3: Baixando assets...`);
|
||
|
||
const replacements = new Map();
|
||
const batchSize = 5; // Reduzido para evitar sobrecarga
|
||
const assetEntries = [...globalAssets.entries()];
|
||
let successCount = 0;
|
||
let failCount = 0;
|
||
|
||
for (let i = 0; i < assetEntries.length; i += batchSize) {
|
||
const batch = assetEntries.slice(i, i + batchSize);
|
||
const promises = batch.map(async ([originalUrl, fullUrl]) => {
|
||
const info = this.getAssetInfo(fullUrl, startUrl, siteDir, assetFiles);
|
||
if (info) {
|
||
const ok = await this.downloadAsset(info.fullUrl, info.fullPath);
|
||
if (ok) {
|
||
replacements.set(originalUrl, info.relativePath);
|
||
replacements.set(info.fullUrl, info.relativePath);
|
||
|
||
// Variações de protocolo
|
||
if (info.fullUrl.startsWith('https://')) {
|
||
replacements.set(info.fullUrl.replace('https://', 'http://'), info.relativePath);
|
||
} else if (info.fullUrl.startsWith('http://')) {
|
||
replacements.set(info.fullUrl.replace('http://', 'https://'), info.relativePath);
|
||
}
|
||
|
||
successCount++;
|
||
return { ok: true, info, fullUrl };
|
||
} else {
|
||
failCount++;
|
||
}
|
||
}
|
||
return { ok: false };
|
||
});
|
||
|
||
await Promise.all(promises);
|
||
|
||
const progress = Math.round((i + batch.length) / assetEntries.length * 100);
|
||
console.log(` 📊 Progresso: ${progress}% (${successCount} ok, ${failCount} falhas)`);
|
||
}
|
||
|
||
console.log(`\n✅ ${successCount} assets baixados com sucesso\n`);
|
||
|
||
// Fase 4: Extrair assets de CSS
|
||
console.log(`🎨 FASE 4: Extraindo assets de arquivos CSS...`);
|
||
|
||
const cssAssets = new Map();
|
||
let cssAssetCount = 0;
|
||
|
||
for (const [original, local] of replacements) {
|
||
if (local.endsWith('.css')) {
|
||
try {
|
||
const cssPath = path.join(siteDir, local);
|
||
if (fs.existsSync(cssPath)) {
|
||
const cssContent = fs.readFileSync(cssPath, 'utf8');
|
||
|
||
// Determinar URL base do CSS
|
||
let cssBaseUrl = original;
|
||
for (const [origUrl, localPath] of replacements) {
|
||
if (localPath === local) {
|
||
cssBaseUrl = origUrl;
|
||
break;
|
||
}
|
||
}
|
||
|
||
const cssUrls = this.extractUrlsFromCSS(cssContent, cssBaseUrl);
|
||
|
||
for (const cssUrl of cssUrls) {
|
||
if (!replacements.has(cssUrl.full) && !cssAssets.has(cssUrl.full)) {
|
||
cssAssets.set(cssUrl.original, cssUrl.full);
|
||
}
|
||
}
|
||
}
|
||
} catch(e) {}
|
||
}
|
||
}
|
||
|
||
if (cssAssets.size > 0) {
|
||
console.log(` 📦 Encontrados ${cssAssets.size} assets em CSS (fontes, imagens, etc.)`);
|
||
|
||
for (const [originalUrl, fullUrl] of cssAssets) {
|
||
const info = this.getAssetInfo(fullUrl, startUrl, siteDir, assetFiles);
|
||
if (info) {
|
||
const ok = await this.downloadAsset(info.fullUrl, info.fullPath);
|
||
if (ok) {
|
||
replacements.set(originalUrl, info.relativePath);
|
||
replacements.set(info.fullUrl, info.relativePath);
|
||
cssAssetCount++;
|
||
|
||
// Atualizar CSS files para substituir URLs
|
||
for (const [origCss, localCss] of replacements) {
|
||
if (localCss.endsWith('.css')) {
|
||
try {
|
||
const cssPath = path.join(siteDir, localCss);
|
||
if (fs.existsSync(cssPath)) {
|
||
let cssContent = fs.readFileSync(cssPath, 'utf8');
|
||
const escaped = originalUrl.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||
cssContent = cssContent.replace(new RegExp(escaped, 'g'), info.relativePath);
|
||
fs.writeFileSync(cssPath, cssContent, 'utf8');
|
||
}
|
||
} catch(e) {}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
console.log(` ✅ ${cssAssetCount} assets de CSS baixados`);
|
||
} else {
|
||
console.log(` ℹ️ Nenhum asset adicional encontrado em CSS`);
|
||
}
|
||
|
||
// Fase 5: Processar e salvar páginas
|
||
console.log(`\n💾 FASE 5: Processando e salvando páginas HTML...\n`);
|
||
|
||
const sortedReplacements = [...replacements.entries()]
|
||
.sort((a, b) => b[0].length - a[0].length);
|
||
|
||
const elementsToRemove = [
|
||
'.cookie-banner', '.cookie-notice', '.cookie-consent', '.cookies-banner',
|
||
'.lgpd-banner', '.lgpd-notice', '.lgpd-popup', '.lgpd',
|
||
'#cookie-banner', '#cookie-notice', '#cookie-consent', '#cookies',
|
||
'#lgpd-banner', '#lgpd', '#lgpd-notice',
|
||
'[class*="cookie"]', '[class*="lgpd"]', '[class*="gdpr"]',
|
||
'[id*="cookie"]', '[id*="lgpd"]', '[id*="gdpr"]',
|
||
'.popup-overlay', '.modal-overlay', '.consent-banner',
|
||
'[class*="consent"]', '[class*="privacy-banner"]',
|
||
'.joinchat', '#joinchat', '.joinchat__button',
|
||
];
|
||
|
||
let pagesSaved = 0;
|
||
for (const [pageUrl, pageInfo] of pageMap) {
|
||
let html = pageContents.get(pageUrl);
|
||
if (!html) continue;
|
||
|
||
console.log(` 💾 ${pageInfo.file}`);
|
||
|
||
// Remover elementos indesejados
|
||
const $ = cheerio.load(html);
|
||
|
||
for (const selector of elementsToRemove) {
|
||
try { $(selector).remove(); } catch(e) {}
|
||
}
|
||
|
||
// Remover scripts problemáticos e externos que não funcionam offline
|
||
$('script').each((_, el) => {
|
||
const src = $(el).attr('src') || '';
|
||
const content = $(el).html() || '';
|
||
|
||
// Lista de scripts a remover
|
||
const removePatterns = [
|
||
'cookie', 'lgpd', 'consent', 'gdpr',
|
||
'recaptcha', 'google-analytics', 'googletagmanager', 'gtag', 'gtm',
|
||
'facebook.net', 'fbevents', 'fbq',
|
||
'doubleclick.net', 'googleadservices',
|
||
'analytics.js', 'ga.js'
|
||
];
|
||
|
||
const shouldRemove = removePatterns.some(pattern =>
|
||
src.includes(pattern) || content.includes(pattern)
|
||
);
|
||
|
||
if (shouldRemove && !src.includes('jquery') && !src.includes('elementor')) {
|
||
$(el).remove();
|
||
}
|
||
});
|
||
|
||
// Remover iframes externos (Google Tag Manager, etc)
|
||
$('iframe[src*="googletagmanager"], iframe[src*="facebook"], iframe[src*="doubleclick"]').remove();
|
||
|
||
// Remover noscript tags de tracking
|
||
$('noscript').each((_, el) => {
|
||
const content = $(el).html() || '';
|
||
if (content.includes('googletagmanager') || content.includes('facebook')) {
|
||
$(el).remove();
|
||
}
|
||
});
|
||
|
||
html = $.html();
|
||
|
||
// Substituir assets - usar substituição mais precisa
|
||
for (const [original, local] of sortedReplacements) {
|
||
// Escapar caracteres especiais
|
||
const escaped = original.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||
|
||
// Substituir em diferentes contextos
|
||
// 1. Em atributos HTML (src, href, data-src, etc)
|
||
html = html.replace(new RegExp(`(src|href|data-src|data-lazy-src|data-background|poster)=["']${escaped}["']`, 'gi'), `$1="${local}"`);
|
||
|
||
// 2. Em CSS inline (background-image, etc)
|
||
html = html.replace(new RegExp(`url\\(['"]?${escaped}['"]?\\)`, 'gi'), `url("${local}")`);
|
||
|
||
// 3. Substituição direta
|
||
html = html.replace(new RegExp(escaped, 'g'), local);
|
||
}
|
||
|
||
// Substituir links internos
|
||
for (const [url, info] of pageMap) {
|
||
const escaped = url.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||
html = html.replace(new RegExp(`href=["']${escaped}["']`, 'gi'), `href="${info.file}"`);
|
||
|
||
const urlObj = new URL(url);
|
||
if (urlObj.pathname && urlObj.pathname !== '/') {
|
||
const pathname = urlObj.pathname.replace(/\/$/, '');
|
||
html = html.replace(
|
||
new RegExp(`href=["']${pathname.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}["']`, 'gi'),
|
||
`href="${info.file}"`
|
||
);
|
||
}
|
||
}
|
||
|
||
// Links restantes do domínio -> index.html
|
||
html = html.replace(
|
||
new RegExp(`href=["']https?://(www\\.)?${baseHostClean.replace(/\./g, '\\.')}[^"']*["']`, 'gi'),
|
||
'href="index.html"'
|
||
);
|
||
|
||
// Adicionar script para suprimir erros e melhorar compatibilidade offline
|
||
const offlineScript = `
|
||
<script>
|
||
// Suprimir erros de scripts externos
|
||
window.addEventListener('error', function(e) {
|
||
// Ignorar erros de recursos externos
|
||
if (e.filename && (
|
||
e.filename.includes('recaptcha') ||
|
||
e.filename.includes('google') ||
|
||
e.filename.includes('facebook') ||
|
||
e.filename.includes('doubleclick')
|
||
)) {
|
||
e.preventDefault();
|
||
return true;
|
||
}
|
||
}, true);
|
||
|
||
// Criar stubs para funções que podem estar faltando
|
||
window.fbq = window.fbq || function() { console.log('fbq stub called'); };
|
||
window.gtag = window.gtag || function() { console.log('gtag stub called'); };
|
||
window.dataLayer = window.dataLayer || [];
|
||
window.ga = window.ga || function() { console.log('ga stub called'); };
|
||
|
||
// Desabilitar reCAPTCHA se presente
|
||
if (typeof grecaptcha !== 'undefined') {
|
||
grecaptcha.ready = function(cb) { console.log('reCAPTCHA disabled offline'); };
|
||
}
|
||
|
||
console.log('🌐 Site clonado carregado em modo offline');
|
||
</script>
|
||
`;
|
||
|
||
// Inserir antes do </body>
|
||
html = html.replace('</body>', offlineScript + '</body>');
|
||
|
||
fs.writeFileSync(path.join(siteDir, pageInfo.file), html, 'utf8');
|
||
pagesSaved++;
|
||
}
|
||
|
||
// Salvar informações
|
||
const info = {
|
||
originalUrl: startUrl,
|
||
clonedAt: new Date().toISOString(),
|
||
pages: [...pageMap.entries()].map(([url, info]) => ({ url, file: info.file })),
|
||
assetsDownloaded: successCount + cssAssetCount,
|
||
assetsFailed: failCount,
|
||
totalAssets: globalAssets.size + cssAssets.size,
|
||
directory: siteDir
|
||
};
|
||
|
||
fs.writeFileSync(path.join(siteDir, 'clone-info.json'), JSON.stringify(info, null, 2));
|
||
fs.writeFileSync(path.join(siteDir, 'abrir-site.bat'), `@echo off\nstart index.html`);
|
||
|
||
console.log(`\n${'='.repeat(60)}`);
|
||
console.log(`🎉 CLONAGEM CONCLUÍDA COM SUCESSO!`);
|
||
console.log(`${'='.repeat(60)}`);
|
||
console.log(`📄 Páginas clonadas: ${pagesSaved}`);
|
||
console.log(`📦 Assets baixados: ${successCount + cssAssetCount}`);
|
||
console.log(`⚠️ Assets com falha: ${failCount}`);
|
||
console.log(`📁 Localização: ${siteDir}`);
|
||
console.log(`${'='.repeat(60)}\n`);
|
||
|
||
return {
|
||
url: startUrl,
|
||
title: 'Site Clonado',
|
||
statusCode: response.status(),
|
||
links: [],
|
||
images: [],
|
||
contentLength: 0,
|
||
timestamp: new Date().toISOString(),
|
||
cloneInfo: {
|
||
directory: siteDir,
|
||
pagesCloned: pagesSaved,
|
||
assetsDownloaded: successCount + cssAssetCount,
|
||
assetsFailed: failCount,
|
||
totalAssets: globalAssets.size + cssAssets.size,
|
||
success: true
|
||
}
|
||
};
|
||
|
||
} finally {
|
||
await browser.close();
|
||
}
|
||
}
|
||
}
|
||
|
||
// API
|
||
app.post('/crawl', async (req, res) => {
|
||
try {
|
||
const { url } = req.body;
|
||
if (!url) return res.status(400).json({ error: 'URL obrigatória' });
|
||
|
||
const cloner = new AdvancedWebCloner();
|
||
const result = await cloner.cloneWebsite(url);
|
||
res.json({ success: true, data: result });
|
||
} catch (error) {
|
||
console.error('❌ Erro:', error.message);
|
||
res.status(500).json({ success: false, error: error.message });
|
||
}
|
||
});
|
||
|
||
app.get('/health', (req, res) => {
|
||
res.json({ status: 'ok', service: 'advanced-web-cloner' });
|
||
});
|
||
|
||
app.listen(3001, () => {
|
||
console.log('🌐 Advanced Web Cloner rodando em http://localhost:3001');
|
||
console.log('🚀 Pronto para clonagem profunda de sites!\n');
|
||
});
|