const fs = require('fs'); const path = require('path'); const { chromium } = require('playwright'); const { URL } = require('url'); const aiService = require('./ai-recreation'); class SmartCloner { constructor() { this.baseOutputDir = path.join(__dirname, '..', 'cloned-sites'); this.ensureDirectoryExists(this.baseOutputDir); this.resourceMap = new Map(); // Map } ensureDirectoryExists(dirPath) { if (!fs.existsSync(dirPath)) { fs.mkdirSync(dirPath, { recursive: true }); } } sanitizeFilename(filename) { if (!filename) return 'unnamed'; return filename.replace(/[<>:"/\\|?*]/g, '_').replace(/\s+/g, '_').substring(0, 200); } generateLocalPath(url, contentType, baseDir) { try { const urlObj = new URL(url); let pathname = urlObj.pathname; if (pathname.endsWith('/') || pathname === '') { const isAsset = /\.(js|css|png|jpg|jpeg|gif|svg|woff2?|ttf|otf)$/i.test(pathname.slice(0, -1)); if (!isAsset) { pathname += 'index.html'; } else { pathname = pathname.slice(0, -1); // Remove trailing slash if it's an asset } } // If we have a contentType, try to get the correct extension let ext = path.extname(pathname); if (!ext) { if (contentType) { if (contentType.includes('text/html')) ext = '.html'; else if (contentType.includes('text/css')) ext = '.css'; else if (contentType.includes('javascript')) ext = '.js'; else if (contentType.includes('image/png')) ext = '.png'; else if (contentType.includes('image/jpeg')) ext = '.jpg'; else if (contentType.includes('image/gif')) ext = '.gif'; else if (contentType.includes('image/svg')) ext = '.svg'; else if (contentType.includes('application/json')) ext = '.json'; else if (contentType.includes('font/')) { if (contentType.includes('woff2')) ext = '.woff2'; else if (contentType.includes('woff')) ext = '.woff'; else if (contentType.includes('ttf')) ext = '.ttf'; } } // Final fallback if still no extension and looks like a route if (!ext && !pathname.includes('.')) { ext = '.html'; } if (ext) pathname += ext; } // Sanitize the path for Windows const segments = pathname.split('/').filter(s => s).map(p => this.sanitizeFilename(p)); return path.join(baseDir, ...segments); } catch (e) { return path.join(baseDir, 'assets', `resource_${Date.now()}`); } } async cloneSite(url, maxPages = 20) { console.log(`\n🚀 INICIANDO DEEP SMART CLONE: ${url}`); console.log(`📂 Limite de páginas: ${maxPages}\n`); this.resourceMap.clear(); let browser = null; try { const urlObj = new URL(url); const siteName = this.sanitizeFilename(urlObj.hostname); const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); const siteDir = path.join(this.baseOutputDir, `${siteName}_${timestamp}`); this.ensureDirectoryExists(siteDir); browser = await chromium.launch({ headless: true, args: ['--no-sandbox', '--disable-setuid-sandbox', '--window-size=1920,1080'] }); const pagesToVisit = [url]; const visitedPages = new Set(); const pageMap = new Map(); // URL -> LocalFilename const apiMocks = []; // Array of { url, method, status, headers, body } // Pre-seed the map so links can be rewritten to files we haven't visited yet pageMap.set(url.replace(/\/$/, ''), 'index.html'); let pageCount = 0; while (pagesToVisit.length > 0 && pageCount < maxPages) { const currentUrl = pagesToVisit.shift(); const cleanCurrentUrl = currentUrl.replace(/\/$/, ''); if (visitedPages.has(cleanCurrentUrl)) continue; pageCount++; visitedPages.add(cleanCurrentUrl); console.log(`📄 [${pageCount}/${maxPages}] Clonando: ${currentUrl}`); const context = await browser.newContext({ viewport: { width: 1920, height: 1080 }}); const page = await context.newPage(); // Asset and API interception page.on('response', async (response) => { const rUrl = response.url(); const req = response.request(); if (rUrl.startsWith('data:') || !response.ok()) return; const resourceType = req.resourceType(); if (resourceType === 'fetch' || resourceType === 'xhr') { try { const bodyStr = await response.text(); apiMocks.push({ url: rUrl, method: req.method(), status: response.status(), headers: response.headers(), body: bodyStr }); } catch (e) { } return; } if (req.method() !== 'GET') return; try { const headers = response.headers(); const contentType = headers['content-type']; const buffer = await response.body(); const localPath = this.generateLocalPath(rUrl, contentType, siteDir); this.ensureDirectoryExists(path.dirname(localPath)); fs.writeFileSync(localPath, buffer); this.resourceMap.set(rUrl, path.relative(siteDir, localPath).replace(/\\/g, '/')); } catch (e) { } }); try { await page.goto(currentUrl, { waitUntil: 'networkidle', timeout: 60000 }); } catch (e) { console.log(`⚠️ Falha ao carregar ${currentUrl}: ${e.message}`); await context.close(); continue; } await this.autoScroll(page); await new Promise(r => setTimeout(r, 1000)); // Discover and map internal links BEFORE rewriting const discovered = await page.evaluate((domain) => { return Array.from(document.querySelectorAll('a[href]')) .map(a => a.href.split('#')[0].replace(/\/$/, '')) .filter(href => href.startsWith(window.location.origin) && !href.includes('wp-admin') && !href.includes('login')); }, urlObj.origin); discovered.forEach(link => { if (!pageMap.has(link)) { let name = link.split('/').pop().split('?')[0] || `page_${pageMap.size + 1}`; if (!name.endsWith('.html')) name += '.html'; pageMap.set(link, name); } if (!visitedPages.has(link) && !pagesToVisit.includes(link)) { pagesToVisit.push(link); } }); // Rewrite and Clean const resMapArr = Array.from(this.resourceMap.entries()); const pagMapArr = Array.from(pageMap.entries()); await page.evaluate((resArr, pagArr) => { const rMap = new Map(resArr); const pMap = new Map(pagArr); function getLocal(url) { if (!url) return null; try { const full = new URL(url, document.baseURI).href.split('#')[0].replace(/\/$/, ''); if (rMap.has(full)) return rMap.get(full); if (pMap.has(full)) return pMap.get(full); return null; } catch (e) { return null; } } // 1. Rewrite Links, Images, Scripts document.querySelectorAll('[src], [href], [data-src], [srcset]').forEach(el => { ['src', 'href', 'data-src'].forEach(attr => { if (el.hasAttribute(attr)) { const local = getLocal(el.getAttribute(attr)); if (local) el.setAttribute(attr, local); } }); if (el.hasAttribute('srcset')) { const parts = el.getAttribute('srcset').split(','); const mapped = parts.map(p => { const [u, ...rest] = p.trim().split(/\s+/); const l = getLocal(u); return l ? [l, ...rest].join(' ') : p; }); el.setAttribute('srcset', mapped.join(', ')); } }); // 2. Clear common barriers const problematic = ['script[src*="analytics"]', 'script[src*="facebook"]', 'iframe[src*="google.com/maps"]', '.cookie-banner', '.modal-backdrop']; problematic.forEach(s => document.querySelectorAll(s).forEach(e => e.remove())); // 3. Smart Fixer Script const script = document.createElement('script'); script.textContent = ` (function() { function fix() { document.documentElement.style.setProperty('overflow', 'auto', 'important'); document.body.style.setProperty('overflow', 'auto', 'important'); document.body.style.setProperty('pointer-events', 'auto', 'important'); const loaders = ['#preloader', '.loader', '.loading-overlay', '.elementor-loader']; loaders.forEach(s => document.querySelectorAll(s).forEach(e => e.remove())); } fix(); setInterval(fix, 2000); window.fbq = window.gtag = function(){}; })(); `; document.body.appendChild(script); // 4. Inject Service Worker Registration const swScript = document.createElement('script'); swScript.textContent = ` if ('serviceWorker' in navigator) { window.addEventListener('load', function() { navigator.serviceWorker.register('/mock-sw.js').then(function(registration) { console.log('Mock ServiceWorker registration successful with scope: ', registration.scope); }, function(err) { console.log('Mock ServiceWorker registration failed: ', err); }); }); } `; document.body.appendChild(swScript); }, resMapArr, pagMapArr); // 5. AI Recreation of forms and dynamic content const dynamicElements = await page.evaluate(() => { return Array.from(document.querySelectorAll('form, [data-dynamic], .dynamic-content')).map(f => ({ html: f.outerHTML, tagName: f.tagName })); }); for (const el of dynamicElements) { const newHtml = await aiService.recreateComponent(el.html, el.tagName); if (newHtml) { await page.evaluate(({newHtml, oldHtml}) => { const nodes = Array.from(document.querySelectorAll('form, [data-dynamic], .dynamic-content')); const target = nodes.find(n => n.outerHTML === oldHtml); if (target) { target.outerHTML = newHtml; } }, { newHtml, oldHtml: el.html }); } } // 6. Expand Shadow DOM to Declarative Shadow DOM await page.evaluate(() => { function expandShadowRoots(node) { if (node.shadowRoot) { const template = document.createElement('template'); template.setAttribute('shadowrootmode', 'open'); template.innerHTML = node.shadowRoot.innerHTML; node.insertBefore(template, node.firstChild); // Recursively process inside shadowRoot Array.from(node.shadowRoot.querySelectorAll('*')).forEach(expandShadowRoots); } Array.from(node.children).forEach(expandShadowRoots); } expandShadowRoots(document.body); }); const html = await page.content(); const localFile = pageMap.get(cleanCurrentUrl) || 'index.html'; fs.writeFileSync(path.join(siteDir, localFile), html); await context.close(); } const info = { url, timestamp: new Date().toISOString(), pages: pageCount, directory: siteDir }; fs.writeFileSync(path.join(siteDir, 'clone-info.json'), JSON.stringify(info, null, 2)); // Save Mock DB fs.writeFileSync(path.join(siteDir, 'mock-db.json'), JSON.stringify(apiMocks, null, 2)); // Create Service Worker const swCode = ` const MOCK_DB_URL = '/mock-db.json'; let mockDb = null; self.addEventListener('install', (event) => { self.skipWaiting(); }); self.addEventListener('activate', (event) => { event.waitUntil(clients.claim()); event.waitUntil( fetch(MOCK_DB_URL).then(res => res.json()).then(data => { mockDb = data; }) ); }); self.addEventListener('fetch', (event) => { if (event.request.method === 'GET' && !event.request.url.includes('/mock-db.json') && mockDb) { const mock = mockDb.find(m => m.url === event.request.url || event.request.url.includes(m.url)); if (mock) { event.respondWith( new Response(mock.body, { status: mock.status, headers: { 'Content-Type': mock.headers['content-type'] || 'application/json' } }) ); return; } } }); `; fs.writeFileSync(path.join(siteDir, 'mock-sw.js'), swCode); var countFiles = function(dir) { var ts=0,fc=0; (function rec(d){if(!fs.existsSync(d))return;fs.readdirSync(d).forEach(function(i){var p=path.join(d,i),s=fs.statSync(p);if(s.isDirectory()&&i.indexOf("cloned-sites")==-1&&i.indexOf(".versions")==-1)rec(p);else if(!s.isDirectory()){ts+=s.size;fc++;}});})(dir);return{fileCount:fc,totalSize:ts};}; const stats = countFiles(siteDir); return { success: true, path: siteDir, pagesCloned: pageCount, files: stats.fileCount, size: stats.totalSize }; } catch (e) { console.error('❌ Erro Deep Clone:', e); return { success: false, error: e.message }; } finally { if (browser) await browser.close(); } } async autoScroll(page) { await page.evaluate(async () => { await new Promise((resolve) => { let totalHeight = 0; const distance = 100; const timer = setInterval(() => { const scrollHeight = document.body.scrollHeight; window.scrollBy(0, distance); totalHeight += distance; if (totalHeight >= scrollHeight - window.innerHeight) { clearInterval(timer); resolve(); } }, 100); }); }); } } if (require.main === module) { const args = process.argv.slice(2); const urlArg = args.find(arg => arg.startsWith('--url=')); const url = urlArg ? urlArg.split('=')[1] : args[0]; if (url) { const cloner = new SmartCloner(); cloner.cloneSite(url).then(res => { console.log(JSON.stringify(res, null, 2)); process.exit(res.success ? 0 : 1); }); } else { console.log('Usage: node smart-cloner.js --url='); } } module.exports = SmartCloner;