271 lines
9.9 KiB
JavaScript
271 lines
9.9 KiB
JavaScript
const fs = require('fs');
|
|
const path = require('path');
|
|
const puppeteer = require('puppeteer');
|
|
const { URL } = require('url');
|
|
|
|
class SmartCloner {
|
|
constructor() {
|
|
this.baseOutputDir = path.join(__dirname, '..', 'cloned-sites');
|
|
this.ensureDirectoryExists(this.baseOutputDir);
|
|
this.resourceMap = new Map(); // Map<Url, LocalPath>
|
|
}
|
|
|
|
ensureDirectoryExists(dirPath) {
|
|
if (!fs.existsSync(dirPath)) {
|
|
fs.mkdirSync(dirPath, { recursive: true });
|
|
}
|
|
}
|
|
|
|
sanitizeFilename(filename) {
|
|
if (!filename) return 'unnamed';
|
|
return filename.replace(/[<>:"/\\|?*]/g, '_').replace(/\s+/g, '_').substring(0, 200);
|
|
}
|
|
|
|
generateLocalPath(url, contentType, baseDir) {
|
|
try {
|
|
const urlObj = new URL(url);
|
|
let pathname = urlObj.pathname;
|
|
|
|
if (pathname.endsWith('/') || pathname === '') {
|
|
const isAsset = /\.(js|css|png|jpg|jpeg|gif|svg|woff2?|ttf|otf)$/i.test(pathname.slice(0, -1));
|
|
if (!isAsset) {
|
|
pathname += 'index.html';
|
|
} else {
|
|
pathname = pathname.slice(0, -1); // Remove trailing slash if it's an asset
|
|
}
|
|
}
|
|
|
|
// If we have a contentType, try to get the correct extension
|
|
let ext = path.extname(pathname);
|
|
if (!ext) {
|
|
if (contentType) {
|
|
if (contentType.includes('text/html')) ext = '.html';
|
|
else if (contentType.includes('text/css')) ext = '.css';
|
|
else if (contentType.includes('javascript')) ext = '.js';
|
|
else if (contentType.includes('image/png')) ext = '.png';
|
|
else if (contentType.includes('image/jpeg')) ext = '.jpg';
|
|
else if (contentType.includes('image/gif')) ext = '.gif';
|
|
else if (contentType.includes('image/svg')) ext = '.svg';
|
|
else if (contentType.includes('application/json')) ext = '.json';
|
|
else if (contentType.includes('font/')) {
|
|
if (contentType.includes('woff2')) ext = '.woff2';
|
|
else if (contentType.includes('woff')) ext = '.woff';
|
|
else if (contentType.includes('ttf')) ext = '.ttf';
|
|
}
|
|
}
|
|
|
|
// Final fallback if still no extension and looks like a route
|
|
if (!ext && !pathname.includes('.')) {
|
|
ext = '.html';
|
|
}
|
|
|
|
if (ext) pathname += ext;
|
|
}
|
|
|
|
// Sanitize the path for Windows
|
|
const segments = pathname.split('/').filter(s => s).map(p => this.sanitizeFilename(p));
|
|
return path.join(baseDir, ...segments);
|
|
} catch (e) {
|
|
return path.join(baseDir, 'assets', `resource_${Date.now()}`);
|
|
}
|
|
}
|
|
|
|
async cloneSite(url, maxPages = 20) {
|
|
console.log(`\n🚀 INICIANDO DEEP SMART CLONE: ${url}`);
|
|
console.log(`📂 Limite de páginas: ${maxPages}\n`);
|
|
|
|
this.resourceMap.clear();
|
|
let browser = null;
|
|
|
|
try {
|
|
const urlObj = new URL(url);
|
|
const siteName = this.sanitizeFilename(urlObj.hostname);
|
|
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
|
const siteDir = path.join(this.baseOutputDir, `${siteName}_${timestamp}`);
|
|
this.ensureDirectoryExists(siteDir);
|
|
|
|
browser = await puppeteer.launch({
|
|
headless: 'new',
|
|
args: ['--no-sandbox', '--disable-setuid-sandbox', '--window-size=1920,1080']
|
|
});
|
|
|
|
const pagesToVisit = [url];
|
|
const visitedPages = new Set();
|
|
const pageMap = new Map(); // URL -> LocalFilename
|
|
|
|
// Pre-seed the map so links can be rewritten to files we haven't visited yet
|
|
pageMap.set(url.replace(/\/$/, ''), 'index.html');
|
|
|
|
let pageCount = 0;
|
|
while (pagesToVisit.length > 0 && pageCount < maxPages) {
|
|
const currentUrl = pagesToVisit.shift();
|
|
const cleanCurrentUrl = currentUrl.replace(/\/$/, '');
|
|
|
|
if (visitedPages.has(cleanCurrentUrl)) continue;
|
|
|
|
pageCount++;
|
|
visitedPages.add(cleanCurrentUrl);
|
|
|
|
console.log(`📄 [${pageCount}/${maxPages}] Clonando: ${currentUrl}`);
|
|
|
|
const page = await browser.newPage();
|
|
await page.setViewport({ width: 1920, height: 1080 });
|
|
|
|
// Asset interception
|
|
page.on('response', async (response) => {
|
|
const rUrl = response.url();
|
|
if (rUrl.startsWith('data:') || !response.ok() || response.request().method() !== 'GET') return;
|
|
try {
|
|
const contentType = response.headers()['content-type'];
|
|
const buffer = await response.buffer();
|
|
const localPath = this.generateLocalPath(rUrl, contentType, siteDir);
|
|
this.ensureDirectoryExists(path.dirname(localPath));
|
|
fs.writeFileSync(localPath, buffer);
|
|
this.resourceMap.set(rUrl, path.relative(siteDir, localPath).replace(/\\/g, '/'));
|
|
} catch (e) { }
|
|
});
|
|
|
|
try {
|
|
await page.goto(currentUrl, { waitUntil: 'networkidle2', timeout: 60000 });
|
|
} catch (e) {
|
|
console.log(`⚠️ Falha ao carregar ${currentUrl}: ${e.message}`);
|
|
await page.close();
|
|
continue;
|
|
}
|
|
|
|
await this.autoScroll(page);
|
|
await new Promise(r => setTimeout(r, 1000));
|
|
|
|
// Discover and map internal links BEFORE rewriting
|
|
const discovered = await page.evaluate((domain) => {
|
|
return Array.from(document.querySelectorAll('a[href]'))
|
|
.map(a => a.href.split('#')[0].replace(/\/$/, ''))
|
|
.filter(href => href.startsWith(window.location.origin) && !href.includes('wp-admin') && !href.includes('login'));
|
|
}, urlObj.origin);
|
|
|
|
discovered.forEach(link => {
|
|
if (!pageMap.has(link)) {
|
|
let name = link.split('/').pop().split('?')[0] || `page_${pageMap.size + 1}`;
|
|
if (!name.endsWith('.html')) name += '.html';
|
|
pageMap.set(link, name);
|
|
}
|
|
if (!visitedPages.has(link) && !pagesToVisit.includes(link)) {
|
|
pagesToVisit.push(link);
|
|
}
|
|
});
|
|
|
|
// Rewrite and Clean
|
|
const resMapArr = Array.from(this.resourceMap.entries());
|
|
const pagMapArr = Array.from(pageMap.entries());
|
|
|
|
await page.evaluate((resArr, pagArr) => {
|
|
const rMap = new Map(resArr);
|
|
const pMap = new Map(pagArr);
|
|
|
|
function getLocal(url) {
|
|
if (!url) return null;
|
|
try {
|
|
const full = new URL(url, document.baseURI).href.split('#')[0].replace(/\/$/, '');
|
|
if (rMap.has(full)) return rMap.get(full);
|
|
if (pMap.has(full)) return pMap.get(full);
|
|
return null;
|
|
} catch (e) { return null; }
|
|
}
|
|
|
|
// 1. Rewrite Links, Images, Scripts
|
|
document.querySelectorAll('[src], [href], [data-src], [srcset]').forEach(el => {
|
|
['src', 'href', 'data-src'].forEach(attr => {
|
|
if (el.hasAttribute(attr)) {
|
|
const local = getLocal(el.getAttribute(attr));
|
|
if (local) el.setAttribute(attr, local);
|
|
}
|
|
});
|
|
|
|
if (el.hasAttribute('srcset')) {
|
|
const parts = el.getAttribute('srcset').split(',');
|
|
const mapped = parts.map(p => {
|
|
const [u, ...rest] = p.trim().split(/\s+/);
|
|
const l = getLocal(u);
|
|
return l ? [l, ...rest].join(' ') : p;
|
|
});
|
|
el.setAttribute('srcset', mapped.join(', '));
|
|
}
|
|
});
|
|
|
|
// 2. Clear common barriers
|
|
const problematic = ['script[src*="analytics"]', 'script[src*="facebook"]', 'iframe[src*="google.com/maps"]', '.cookie-banner', '.modal-backdrop'];
|
|
problematic.forEach(s => document.querySelectorAll(s).forEach(e => e.remove()));
|
|
|
|
// 3. Smart Fixer Script
|
|
const script = document.createElement('script');
|
|
script.textContent = `
|
|
(function() {
|
|
function fix() {
|
|
document.documentElement.style.setProperty('overflow', 'auto', 'important');
|
|
document.body.style.setProperty('overflow', 'auto', 'important');
|
|
document.body.style.setProperty('pointer-events', 'auto', 'important');
|
|
const loaders = ['#preloader', '.loader', '.loading-overlay', '.elementor-loader'];
|
|
loaders.forEach(s => document.querySelectorAll(s).forEach(e => e.remove()));
|
|
}
|
|
fix(); setInterval(fix, 2000);
|
|
window.fbq = window.gtag = function(){};
|
|
})();
|
|
`;
|
|
document.body.appendChild(script);
|
|
}, resMapArr, pagMapArr);
|
|
|
|
const html = await page.content();
|
|
const localFile = pageMap.get(cleanCurrentUrl) || 'index.html';
|
|
fs.writeFileSync(path.join(siteDir, localFile), html);
|
|
await page.close();
|
|
}
|
|
|
|
const info = { url, timestamp: new Date().toISOString(), pages: pageCount, directory: siteDir };
|
|
fs.writeFileSync(path.join(siteDir, 'clone-info.json'), JSON.stringify(info, null, 2));
|
|
|
|
return { success: true, path: siteDir, pagesCloned: pageCount };
|
|
} catch (e) {
|
|
console.error('❌ Erro Deep Clone:', e);
|
|
return { success: false, error: e.message };
|
|
} finally {
|
|
if (browser) await browser.close();
|
|
}
|
|
}
|
|
|
|
async autoScroll(page) {
|
|
await page.evaluate(async () => {
|
|
await new Promise((resolve) => {
|
|
let totalHeight = 0;
|
|
const distance = 100;
|
|
const timer = setInterval(() => {
|
|
const scrollHeight = document.body.scrollHeight;
|
|
window.scrollBy(0, distance);
|
|
totalHeight += distance;
|
|
if (totalHeight >= scrollHeight - window.innerHeight) {
|
|
clearInterval(timer);
|
|
resolve();
|
|
}
|
|
}, 100);
|
|
});
|
|
});
|
|
}
|
|
}
|
|
|
|
if (require.main === module) {
|
|
const args = process.argv.slice(2);
|
|
const urlArg = args.find(arg => arg.startsWith('--url='));
|
|
const url = urlArg ? urlArg.split('=')[1] : args[0];
|
|
|
|
if (url) {
|
|
const cloner = new SmartCloner();
|
|
cloner.cloneSite(url).then(res => {
|
|
console.log(JSON.stringify(res, null, 2));
|
|
process.exit(res.success ? 0 : 1);
|
|
});
|
|
} else {
|
|
console.log('Usage: node smart-cloner.js --url=<url>');
|
|
}
|
|
}
|
|
|
|
module.exports = SmartCloner;
|