Hermes
387 lines
15 KiB
JavaScript
387 lines
15 KiB
JavaScript
const fs = require('fs');
|
|
const path = require('path');
|
|
const { chromium } = require('playwright');
|
|
const { URL } = require('url');
|
|
const aiService = require('./ai-recreation');
|
|
|
|
class SmartCloner {
|
|
constructor() {
|
|
this.baseOutputDir = path.join(__dirname, '..', 'cloned-sites');
|
|
this.ensureDirectoryExists(this.baseOutputDir);
|
|
this.resourceMap = new Map(); // Map<Url, LocalPath>
|
|
}
|
|
|
|
ensureDirectoryExists(dirPath) {
|
|
if (!fs.existsSync(dirPath)) {
|
|
fs.mkdirSync(dirPath, { recursive: true });
|
|
}
|
|
}
|
|
|
|
sanitizeFilename(filename) {
|
|
if (!filename) return 'unnamed';
|
|
return filename.replace(/[<>:"/\\|?*]/g, '_').replace(/\s+/g, '_').substring(0, 200);
|
|
}
|
|
|
|
generateLocalPath(url, contentType, baseDir) {
|
|
try {
|
|
const urlObj = new URL(url);
|
|
let pathname = urlObj.pathname;
|
|
|
|
if (pathname.endsWith('/') || pathname === '') {
|
|
const isAsset = /\.(js|css|png|jpg|jpeg|gif|svg|woff2?|ttf|otf)$/i.test(pathname.slice(0, -1));
|
|
if (!isAsset) {
|
|
pathname += 'index.html';
|
|
} else {
|
|
pathname = pathname.slice(0, -1); // Remove trailing slash if it's an asset
|
|
}
|
|
}
|
|
|
|
// If we have a contentType, try to get the correct extension
|
|
let ext = path.extname(pathname);
|
|
if (!ext) {
|
|
if (contentType) {
|
|
if (contentType.includes('text/html')) ext = '.html';
|
|
else if (contentType.includes('text/css')) ext = '.css';
|
|
else if (contentType.includes('javascript')) ext = '.js';
|
|
else if (contentType.includes('image/png')) ext = '.png';
|
|
else if (contentType.includes('image/jpeg')) ext = '.jpg';
|
|
else if (contentType.includes('image/gif')) ext = '.gif';
|
|
else if (contentType.includes('image/svg')) ext = '.svg';
|
|
else if (contentType.includes('application/json')) ext = '.json';
|
|
else if (contentType.includes('font/')) {
|
|
if (contentType.includes('woff2')) ext = '.woff2';
|
|
else if (contentType.includes('woff')) ext = '.woff';
|
|
else if (contentType.includes('ttf')) ext = '.ttf';
|
|
}
|
|
}
|
|
|
|
// Final fallback if still no extension and looks like a route
|
|
if (!ext && !pathname.includes('.')) {
|
|
ext = '.html';
|
|
}
|
|
|
|
if (ext) pathname += ext;
|
|
}
|
|
|
|
// Sanitize the path for Windows
|
|
const segments = pathname.split('/').filter(s => s).map(p => this.sanitizeFilename(p));
|
|
return path.join(baseDir, ...segments);
|
|
} catch (e) {
|
|
return path.join(baseDir, 'assets', `resource_${Date.now()}`);
|
|
}
|
|
}
|
|
|
|
async cloneSite(url, maxPages = 20, customFolderName = null) {
|
|
console.log(`\n🚀 INICIANDO DEEP SMART CLONE: ${url}`);
|
|
console.log(`📂 Limite de páginas: ${maxPages}\n`);
|
|
|
|
this.resourceMap.clear();
|
|
let browser = null;
|
|
|
|
try {
|
|
const urlObj = new URL(url);
|
|
const siteName = this.sanitizeFilename(urlObj.hostname);
|
|
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
|
const finalFolderName = customFolderName ? this.sanitizeFilename(customFolderName) : `${siteName}_${timestamp}`;
|
|
const siteDir = path.join(this.baseOutputDir, finalFolderName);
|
|
this.ensureDirectoryExists(siteDir);
|
|
|
|
browser = await chromium.launch({
|
|
headless: true,
|
|
args: ['--no-sandbox', '--disable-setuid-sandbox', '--window-size=1920,1080']
|
|
});
|
|
|
|
const pagesToVisit = [url];
|
|
const visitedPages = new Set();
|
|
const pageMap = new Map(); // URL -> LocalFilename
|
|
const apiMocks = []; // Array of { url, method, status, headers, body }
|
|
|
|
// Pre-seed the map so links can be rewritten to files we haven't visited yet
|
|
pageMap.set(url.replace(/\/$/, ''), 'index.html');
|
|
|
|
let pageCount = 0;
|
|
while (pagesToVisit.length > 0 && pageCount < maxPages) {
|
|
const currentUrl = pagesToVisit.shift();
|
|
const cleanCurrentUrl = currentUrl.replace(/\/$/, '');
|
|
|
|
if (visitedPages.has(cleanCurrentUrl)) continue;
|
|
|
|
pageCount++;
|
|
visitedPages.add(cleanCurrentUrl);
|
|
|
|
console.log(`📄 [${pageCount}/${maxPages}] Clonando: ${currentUrl}`);
|
|
|
|
const context = await browser.newContext({ viewport: { width: 1920, height: 1080 }});
|
|
const page = await context.newPage();
|
|
|
|
// Asset and API interception
|
|
page.on('response', async (response) => {
|
|
const rUrl = response.url();
|
|
const req = response.request();
|
|
|
|
if (rUrl.startsWith('data:') || !response.ok()) return;
|
|
|
|
const resourceType = req.resourceType();
|
|
if (resourceType === 'fetch' || resourceType === 'xhr') {
|
|
try {
|
|
const bodyStr = await response.text();
|
|
apiMocks.push({
|
|
url: rUrl,
|
|
method: req.method(),
|
|
status: response.status(),
|
|
headers: response.headers(),
|
|
body: bodyStr
|
|
});
|
|
} catch (e) { }
|
|
return;
|
|
}
|
|
|
|
if (req.method() !== 'GET') return;
|
|
try {
|
|
const headers = response.headers();
|
|
const contentType = headers['content-type'];
|
|
const buffer = await response.body();
|
|
const localPath = this.generateLocalPath(rUrl, contentType, siteDir);
|
|
this.ensureDirectoryExists(path.dirname(localPath));
|
|
fs.writeFileSync(localPath, buffer);
|
|
this.resourceMap.set(rUrl, path.relative(siteDir, localPath).replace(/\\/g, '/'));
|
|
} catch (e) { }
|
|
});
|
|
|
|
try {
|
|
await page.goto(currentUrl, { waitUntil: 'networkidle', timeout: 60000 });
|
|
} catch (e) {
|
|
console.log(`⚠️ Falha ao carregar ${currentUrl}: ${e.message}`);
|
|
await context.close();
|
|
continue;
|
|
}
|
|
|
|
await this.autoScroll(page);
|
|
await new Promise(r => setTimeout(r, 1000));
|
|
|
|
// Discover and map internal links BEFORE rewriting
|
|
const discovered = await page.evaluate((domain) => {
|
|
return Array.from(document.querySelectorAll('a[href]'))
|
|
.map(a => a.href.split('#')[0].replace(/\/$/, ''))
|
|
.filter(href => href.startsWith(window.location.origin) && !href.includes('wp-admin') && !href.includes('login'));
|
|
}, urlObj.origin);
|
|
|
|
discovered.forEach(link => {
|
|
if (!pageMap.has(link)) {
|
|
let name = link.split('/').pop().split('?')[0] || `page_${pageMap.size + 1}`;
|
|
if (!name.endsWith('.html')) name += '.html';
|
|
pageMap.set(link, name);
|
|
}
|
|
if (!visitedPages.has(link) && !pagesToVisit.includes(link)) {
|
|
pagesToVisit.push(link);
|
|
}
|
|
});
|
|
|
|
// Rewrite and Clean
|
|
const resMapArr = Array.from(this.resourceMap.entries());
|
|
const pagMapArr = Array.from(pageMap.entries());
|
|
|
|
await page.evaluate((resArr, pagArr) => {
|
|
const rMap = new Map(resArr);
|
|
const pMap = new Map(pagArr);
|
|
|
|
function getLocal(url) {
|
|
if (!url) return null;
|
|
try {
|
|
const full = new URL(url, document.baseURI).href.split('#')[0].replace(/\/$/, '');
|
|
if (rMap.has(full)) return rMap.get(full);
|
|
if (pMap.has(full)) return pMap.get(full);
|
|
return null;
|
|
} catch (e) { return null; }
|
|
}
|
|
|
|
// 1. Rewrite Links, Images, Scripts
|
|
document.querySelectorAll('[src], [href], [data-src], [srcset]').forEach(el => {
|
|
['src', 'href', 'data-src'].forEach(attr => {
|
|
if (el.hasAttribute(attr)) {
|
|
const local = getLocal(el.getAttribute(attr));
|
|
if (local) el.setAttribute(attr, local);
|
|
}
|
|
});
|
|
|
|
if (el.hasAttribute('srcset')) {
|
|
const parts = el.getAttribute('srcset').split(',');
|
|
const mapped = parts.map(p => {
|
|
const [u, ...rest] = p.trim().split(/\s+/);
|
|
const l = getLocal(u);
|
|
return l ? [l, ...rest].join(' ') : p;
|
|
});
|
|
el.setAttribute('srcset', mapped.join(', '));
|
|
}
|
|
});
|
|
|
|
// 2. Clear common barriers
|
|
const problematic = ['script[src*="analytics"]', 'script[src*="facebook"]', 'iframe[src*="google.com/maps"]', '.cookie-banner', '.modal-backdrop'];
|
|
problematic.forEach(s => document.querySelectorAll(s).forEach(e => e.remove()));
|
|
|
|
// 3. Smart Fixer Script
|
|
const script = document.createElement('script');
|
|
script.textContent = `
|
|
(function() {
|
|
function fix() {
|
|
document.documentElement.style.setProperty('overflow', 'auto', 'important');
|
|
document.body.style.setProperty('overflow', 'auto', 'important');
|
|
document.body.style.setProperty('pointer-events', 'auto', 'important');
|
|
const loaders = ['#preloader', '.loader', '.loading-overlay', '.elementor-loader'];
|
|
loaders.forEach(s => document.querySelectorAll(s).forEach(e => e.remove()));
|
|
}
|
|
fix(); setInterval(fix, 2000);
|
|
window.fbq = window.gtag = function(){};
|
|
})();
|
|
`;
|
|
document.body.appendChild(script);
|
|
|
|
// 4. Inject Service Worker Registration
|
|
const swScript = document.createElement('script');
|
|
swScript.textContent = `
|
|
if ('serviceWorker' in navigator) {
|
|
window.addEventListener('load', function() {
|
|
navigator.serviceWorker.register('/mock-sw.js').then(function(registration) {
|
|
console.log('Mock ServiceWorker registration successful with scope: ', registration.scope);
|
|
}, function(err) {
|
|
console.log('Mock ServiceWorker registration failed: ', err);
|
|
});
|
|
});
|
|
}
|
|
`;
|
|
document.body.appendChild(swScript);
|
|
}, resMapArr, pagMapArr);
|
|
|
|
// 5. AI Recreation of forms and dynamic content
|
|
const dynamicElements = await page.evaluate(() => {
|
|
return Array.from(document.querySelectorAll('form, [data-dynamic], .dynamic-content')).map(f => ({
|
|
html: f.outerHTML,
|
|
tagName: f.tagName
|
|
}));
|
|
});
|
|
|
|
for (const el of dynamicElements) {
|
|
const newHtml = await aiService.recreateComponent(el.html, el.tagName);
|
|
if (newHtml) {
|
|
await page.evaluate(({newHtml, oldHtml}) => {
|
|
const nodes = Array.from(document.querySelectorAll('form, [data-dynamic], .dynamic-content'));
|
|
const target = nodes.find(n => n.outerHTML === oldHtml);
|
|
if (target) {
|
|
target.outerHTML = newHtml;
|
|
}
|
|
}, { newHtml, oldHtml: el.html });
|
|
}
|
|
}
|
|
|
|
// 6. Expand Shadow DOM to Declarative Shadow DOM
|
|
await page.evaluate(() => {
|
|
function expandShadowRoots(node) {
|
|
if (node.shadowRoot) {
|
|
const template = document.createElement('template');
|
|
template.setAttribute('shadowrootmode', 'open');
|
|
template.innerHTML = node.shadowRoot.innerHTML;
|
|
node.insertBefore(template, node.firstChild);
|
|
// Recursively process inside shadowRoot
|
|
Array.from(node.shadowRoot.querySelectorAll('*')).forEach(expandShadowRoots);
|
|
}
|
|
Array.from(node.children).forEach(expandShadowRoots);
|
|
}
|
|
expandShadowRoots(document.body);
|
|
});
|
|
|
|
const html = await page.content();
|
|
const localFile = pageMap.get(cleanCurrentUrl) || 'index.html';
|
|
fs.writeFileSync(path.join(siteDir, localFile), html);
|
|
await context.close();
|
|
}
|
|
|
|
const info = { url, timestamp: new Date().toISOString(), pages: pageCount, directory: siteDir };
|
|
fs.writeFileSync(path.join(siteDir, 'clone-info.json'), JSON.stringify(info, null, 2));
|
|
|
|
// Save Mock DB
|
|
fs.writeFileSync(path.join(siteDir, 'mock-db.json'), JSON.stringify(apiMocks, null, 2));
|
|
|
|
// Create Service Worker
|
|
const swCode = `
|
|
const MOCK_DB_URL = '/mock-db.json';
|
|
let mockDb = null;
|
|
|
|
self.addEventListener('install', (event) => {
|
|
self.skipWaiting();
|
|
});
|
|
|
|
self.addEventListener('activate', (event) => {
|
|
event.waitUntil(clients.claim());
|
|
event.waitUntil(
|
|
fetch(MOCK_DB_URL).then(res => res.json()).then(data => { mockDb = data; })
|
|
);
|
|
});
|
|
|
|
self.addEventListener('fetch', (event) => {
|
|
if (event.request.method === 'GET' && !event.request.url.includes('/mock-db.json') && mockDb) {
|
|
const mock = mockDb.find(m => m.url === event.request.url || event.request.url.includes(m.url));
|
|
if (mock) {
|
|
event.respondWith(
|
|
new Response(mock.body, {
|
|
status: mock.status,
|
|
headers: { 'Content-Type': mock.headers['content-type'] || 'application/json' }
|
|
})
|
|
);
|
|
return;
|
|
}
|
|
}
|
|
});
|
|
`;
|
|
fs.writeFileSync(path.join(siteDir, 'mock-sw.js'), swCode);
|
|
|
|
var countFiles = function(dir) { var ts=0,fc=0; (function rec(d){if(!fs.existsSync(d))return;fs.readdirSync(d).forEach(function(i){var p=path.join(d,i),s=fs.statSync(p);if(s.isDirectory()&&i.indexOf("cloned-sites")==-1&&i.indexOf(".versions")==-1)rec(p);else if(!s.isDirectory()){ts+=s.size;fc++;}});})(dir);return{fileCount:fc,totalSize:ts};};
|
|
const stats = countFiles(siteDir);
|
|
|
|
return { success: true, path: siteDir, pagesCloned: pageCount, files: stats.fileCount, size: stats.totalSize };
|
|
} catch (e) {
|
|
console.error('❌ Erro Deep Clone:', e);
|
|
return { success: false, error: e.message };
|
|
} finally {
|
|
if (browser) await browser.close();
|
|
}
|
|
}
|
|
|
|
async autoScroll(page) {
|
|
await page.evaluate(async () => {
|
|
await new Promise((resolve) => {
|
|
let totalHeight = 0;
|
|
const distance = 100;
|
|
const timer = setInterval(() => {
|
|
const scrollHeight = document.body.scrollHeight;
|
|
window.scrollBy(0, distance);
|
|
totalHeight += distance;
|
|
if (totalHeight >= scrollHeight - window.innerHeight) {
|
|
clearInterval(timer);
|
|
resolve();
|
|
}
|
|
}, 100);
|
|
});
|
|
});
|
|
}
|
|
}
|
|
|
|
if (require.main === module) {
|
|
const args = process.argv.slice(2);
|
|
const urlArg = args.find(arg => arg.startsWith('--url='));
|
|
const url = urlArg ? urlArg.split('=')[1] : args[0];
|
|
const folderArg = args.find(arg => arg.startsWith('--folderName='));
|
|
const folderName = folderArg ? folderArg.split('=')[1] : null;
|
|
|
|
if (url) {
|
|
const cloner = new SmartCloner();
|
|
cloner.cloneSite(url, 20, folderName).then(res => {
|
|
console.log(JSON.stringify(res, null, 2));
|
|
process.exit(res.success ? 0 : 1);
|
|
});
|
|
} else {
|
|
console.log('Usage: node smart-cloner.js --url=<url>');
|
|
}
|
|
}
|
|
|
|
module.exports = SmartCloner;
|