const express = require('express'); const path = require('path'); const cors = require('cors'); const axios = require('axios'); const { exec, spawn } = require('child_process'); const fs = require('fs'); const app = express(); const PORT = process.env.PORT || 3000; // Storage manager routes const storageManager = require('./storage-manager'); app.use('/api/clones', storageManager); // Security headers app.use((req, res, next) => { res.setHeader('X-Content-Type-Options', 'nosniff'); res.setHeader('X-Frame-Options', 'DENY'); res.setHeader('X-XSS-Protection', '1; mode=block'); res.setHeader('Referrer-Policy', 'strict-origin-when-cross-origin'); next(); }); // Middleware app.use(cors()); app.use(express.json()); // Serve static files with proper MIME types app.use(express.static(path.join(__dirname, 'public'), { setHeaders: (res, path) => { if (path.endsWith('.css')) res.setHeader('Content-Type', 'text/css'); if (path.endsWith('.js')) res.setHeader('Content-Type', 'application/javascript'); if (path.endsWith('.html')) res.setHeader('Content-Type', 'text/html'); } })); // Middleware para injetar fix de roteador em sites clonados (SPAs como React, Vue, Angular, Vite) e reescrever JS app.use((req, res, next) => { if (!req.path.startsWith('/api') && req.path !== '/' && req.path !== '/old') { let urlPath = req.path; // Remover prefixo /cloned-sites se existir para buscar corretamente no disco if (urlPath.startsWith('/cloned-sites/')) { urlPath = urlPath.replace('/cloned-sites/', '/'); } if (!urlPath.match(/\.[a-zA-Z0-9]+$/)) { if (!urlPath.endsWith('/')) urlPath += '/'; urlPath += 'index.html'; } // Interceptação de arquivos HTML (Camada 1 - DOM Mock Universal) if (urlPath.endsWith('.html')) { const filePath = path.join(__dirname, '..', 'cloned-sites', urlPath); if (fs.existsSync(filePath)) { try { let html = fs.readFileSync(filePath, 'utf8'); if (!html.includes('__CLONEWEB_ROUTER_FIX__')) { const injectScript = ` `; if (html.includes('')) { html = html.replace('', '' + injectScript); } else { html = injectScript + html; } } res.setHeader('Content-Type', 'text/html'); return res.send(html); } catch (err) { console.error('Erro ao injetar fix no HTML do clone:', err); } } } // Interceptação de arquivos JS de sites clonados (Camada 2 - JS Rewriter Supremo) if (urlPath.endsWith('.js') && (urlPath.includes('_202') || urlPath.includes('/cloned-sites/'))) { const filePath = path.join(__dirname, '..', 'cloned-sites', urlPath); if (fs.existsSync(filePath)) { try { let jsContent = fs.readFileSync(filePath, 'utf8'); // Substitui chamadas diretas a window.location.pathname e location.pathname jsContent = jsContent.replace(/window\.location\.pathname/g, '(window.__mockLocationPathname || window.location.pathname)'); jsContent = jsContent.replace(/location\.pathname/g, '(window.__mockLocationPathname || window.location.pathname)'); // Substitui chamadas diretas a window.location.href e location.href jsContent = jsContent.replace(/window\.location\.href/g, '(window.__mockLocationHref || window.location.href)'); jsContent = jsContent.replace(/location\.href/g, '(window.__mockLocationHref || window.location.href)'); res.setHeader('Content-Type', 'application/javascript'); return res.send(jsContent); } catch (err) { console.error('Erro ao reescrever JS do clone:', err); } } } } next(); }); // Serve cloned sites directory app.use(express.static(path.join(__dirname, '..', 'cloned-sites'), { setHeaders: (res, filePath) => { if (filePath.endsWith('.css')) res.setHeader('Content-Type', 'text/css'); if (filePath.endsWith('.js')) res.setHeader('Content-Type', 'application/javascript'); } })); // Proxy para o crawler service - SEM TIMEOUT app.post('/api/crawl', async (req, res) => { try { console.log('🚀 Proxying crawl request to crawler service...'); console.log('⏳ Clonagem pode demorar vários minutos para sites grandes...'); // SEM TIMEOUT - permite clonagens longas const response = await axios.post('http://localhost:3001/crawl', req.body, { timeout: 0 // Sem limite de tempo }); console.log('✅ Crawl completed successfully'); res.json(response.data); } catch (error) { console.error('❌ Crawl error:', error.message); res.status(500).json({ success: false, error: error.response?.data?.error || error.message }); } }); // Proxy para o wget cloner - SEM TIMEOUT para clonagens grandes app.post('/api/wget-clone', async (req, res) => { try { console.log('🔧 Proxying wget clone request...'); console.log('⏳ Clonagem pode demorar vários minutos para sites grandes...'); // SEM TIMEOUT - permite clonagens que demoram mais de 10 minutos const response = await axios.post('http://localhost:3002/wget-clone', req.body, { timeout: 0 // Sem limite de tempo }); console.log('✅ Wget clone completed successfully'); // Sync to host /root/Desktop/Clonados const directory = response.data.data?.directory || response.data.directory; if (response.data && response.data.success && directory) { const { exec } = require('child_process'); const containerId = require('os').hostname(); const cloneDir = directory.split('/').pop(); const hostDir = '/root/Desktop/Clonados'; exec(`mkdir -p ${hostDir} && cp -r ${directory} ${hostDir}/ 2>/dev/null || docker cp ${containerId}:${directory} ${hostDir}/ 2>/dev/null`, (err) => { if (err) console.log('⚠️ Sync to host failed:', err.message); else console.log('✅ Clone synced to host Desktop:', cloneDir); }); } res.json(response.data); } catch (error) { console.error('❌ Wget clone error:', error.message); res.status(500).json({ success: false, error: error.response?.data?.error || error.message }); } }); // Smart Clone Endpoint (Local Spawn) app.post('/api/smart-clone', (req, res) => { const { url } = req.body; if (!url) { return res.status(400).json({ error: 'URL is required' }); } console.log('🚀 Starting Smart Clone for:', url); console.log('⏳ This uses Puppeteer and may take a while...'); // Spawn the smart cloner process const scriptPath = path.join(__dirname, '..', 'simple-crawler', 'smart-cloner.js'); console.log(`[SmartClone] Executing: node "${scriptPath}" --url="${url}"`); const child = spawn('node', [scriptPath, `--url=${url}`], { cwd: path.join(__dirname, '..'), shell: true }); let output = ''; let errorOutput = ''; child.stdout.on('data', (data) => { const text = data.toString(); output += text; console.log(`[SmartClone STDOUT] ${text.trim()}`); }); child.stderr.on('data', (data) => { const text = data.toString(); errorOutput += text; console.error(`[SmartClone STDERR] ${text.trim()}`); }); child.on('error', (err) => { console.error('[SmartClone SPAWN ERROR]', err); if (!res.headersSent) { res.status(500).json({ success: false, error: 'Failed to start cloning process', details: err.message }); } }); child.on('close', (code) => { console.log(`[SmartClone] Process exited with code ${code}`); if (res.headersSent) return; try { // Find the last JSON object in output const matches = output.match(/\{[\s\S]*\}/g); if (matches && matches.length > 0) { const lastMatch = matches[matches.length - 1]; const result = JSON.parse(lastMatch); // Sync to host /root/Desktop/Clonados const directory = result.data?.directory || result.directory || result.path; if (result.success && directory) { const { exec } = require('child_process'); const containerId = require('os').hostname(); const cloneDir = directory.split('/').pop(); const hostDir = '/root/Desktop/Clonados'; exec(`mkdir -p ${hostDir} && cp -r ${directory} ${hostDir}/ 2>/dev/null || docker cp ${containerId}:${directory} ${hostDir}/ 2>/dev/null`, (err) => { if (err) console.log('⚠️ SmartClone sync to host failed:', err.message); else console.log('✅ SmartClone synced to host Desktop:', cloneDir); }); } res.json({ success: result.success, data: result, error: result.error }); } else { throw new Error('No valid JSON output found from cloner'); } } catch (e) { console.error('[SmartClone PARSE ERROR]', e.message); if (code === 0) { res.json({ success: true, message: 'Process finished but output parsing failed', fullOutput: output }); } else { res.status(500).json({ success: false, error: 'Clone failed or output was invalid', details: errorOutput }); } } }); }); // Health check do wget cloner (porta 3002) app.get('/api/health', async (req, res) => { try { const response = await axios.get('http://localhost:3002/health', { timeout: 5000 }); res.json(response.data); } catch (error) { res.status(500).json({ status: 'error', message: 'WGET Cloner service unavailable' }); } }); // Rota de visualização de arquivos da pasta clonada (Gerenciador de Arquivos Web Embutido) app.get('/api/browse/:folder', (req, res) => { try { const folderName = req.params.folder; const folderPath = path.join(__dirname, '..', 'cloned-sites', folderName); if (!fs.existsSync(folderPath)) { return res.status(404).send('

Pasta não encontrada

A pasta solicitada não existe no servidor.

'); } const items = fs.readdirSync(folderPath); let filesHtml = ''; items.forEach(item => { const itemPath = path.join(folderPath, item); const stat = fs.statSync(itemPath); const isDir = stat.isDirectory(); const size = isDir ? '-' : (stat.size / 1024).toFixed(2) + ' KB'; const date = new Date(stat.mtime).toLocaleString('pt-BR'); const icon = isDir ? 'fa-folder' : (item.endsWith('.html') ? 'fa-file-code' : (item.endsWith('.json') ? 'fa-file-alt' : 'fa-file')); const link = isDir ? `#` : `/${folderName}/${item}`; filesHtml += ` ${item} ${size} ${date} ${!isDir ? `Visualizar` : ''} ${!isDir ? `Baixar` : ''} `; }); const html = ` Arquivos Clonados - ${folderName}

${folderName}

${filesHtml}
Nome do Ficheiro Tamanho Modificado em Ações
`; res.send(html); } catch (error) { console.error('Erro ao listar pasta:', error); res.status(500).send('

Erro interno do servidor

'); } }); // Abrir pasta do clone (mantém compatibilidade legada e retorna a URL do Web Browse) app.post('/api/open-folder', (req, res) => { try { const { path: folderPath } = req.body; if (!folderPath) { return res.status(400).json({ error: 'Caminho da pasta é obrigatório' }); } const folderName = path.basename(folderPath); const browseUrl = `/api/browse/${folderName}`; res.json({ success: true, message: 'Pasta acessível via Web', url: browseUrl }); } catch (error) { console.error('Erro ao abrir pasta:', error); res.status(500).json({ error: 'Erro interno do servidor' }); } }); // Abrir site clonado via HTTP para evitar problemas de origem/CORS app.post('/api/open-site', (req, res) => { try { const { path: sitePath } = req.body; if (!sitePath) { return res.status(400).json({ error: 'Caminho do site é obrigatório' }); } const folderName = path.basename(sitePath); const siteUrl = `/${folderName}/index.html`; res.json({ success: true, message: 'Site aberto com sucesso', url: siteUrl }); } catch (error) { console.error('Erro ao abrir site:', error); res.status(500).json({ error: 'Erro interno do servidor' }); } }); // Abrir página de navegação via HTTP app.post('/api/open-navigation', (req, res) => { try { const { path: sitePath } = req.body; if (!sitePath) { return res.status(400).json({ error: 'Caminho do site é obrigatório' }); } const folderName = path.basename(sitePath); const navUrl = `http://localhost:${PORT}/${folderName}/navegacao.html`; console.log(`🌍 Opening navigation via HTTP: ${navUrl}`); // Abrir navegacao.html no navegador padrão exec(`start "" "${navUrl}"`, (error) => { if (error) { console.error('Erro ao abrir navegação:', error); return res.status(500).json({ error: 'Não foi possível abrir a navegação via HTTP' }); } res.json({ success: true, message: 'Navegação aberta via HTTP com sucesso', url: navUrl }); }); } catch (error) { console.error('Erro ao abrir navegação:', error); res.status(500).json({ error: 'Erro interno do servidor' }); } }); app.get('/', (req, res) => { res.sendFile(path.join(__dirname, 'public', 'index-new.html')); }); // Serve old interface app.get('/old', (req, res) => { res.sendFile(path.join(__dirname, 'public', 'index.html')); }); // Serve favicon requests app.get('/favicon.ico', (req, res) => { res.redirect('/favicon.svg'); }); // Handle common missing resources app.get('/index', (req, res) => { res.redirect('/'); }); // AI Agent Route (MiniMax) app.post('/api/ai/agent', async (req, res) => { const { systemPrompt, userMessage } = req.body; const apiKey = process.env.MINIMAX_API_KEY; if (!apiKey) { return res.status(500).json({ error: 'MINIMAX_API_KEY is missing in .env' }); } try { const response = await axios.post('https://api.minimax.io/v1/chat/completions', { model: "MiniMax-M2.7", messages: [ { role: "system", content: systemPrompt || "Você é um assistente de inteligência artificial útil." }, { role: "user", content: userMessage } ], max_tokens: 2048 }, { headers: { 'Authorization': `Bearer ${apiKey}`, 'Content-Type': 'application/json' }, timeout: 90000 // 90s }); let reply = response.data.choices?.[0]?.message?.content || 'Processamento concluído, porém sem retorno.'; // Strip reasoning tags if present reply = reply.replace(/[\s\S]*?<\/think>/gi, '').trim(); res.json({ success: true, data: reply }); } catch (error) { console.error('MiniMax AI Error:', error.response?.data || error.message); res.status(500).json({ error: error.message, details: error.response?.data }); } }); // Handle 404s app.use((req, res) => { res.status(404).json({ error: 'Not Found', message: `Route ${req.path} not found` }); }); // Error handler app.use((err, req, res, next) => { console.error('Server error:', err); res.status(500).json({ error: 'Internal Server Error', message: err.message }); }); app.listen(PORT, () => { console.log(`🌐 CloneWeb Interface running on http://localhost:${PORT}`); console.log(`🔗 Make sure crawler service is running on http://localhost:3001`); console.log(`📁 Serving static files from: ${path.join(__dirname, 'public')}`); });