const express = require('express');
const path = require('path');
const cors = require('cors');
const axios = require('axios');
const { exec, spawn } = require('child_process');
const fs = require('fs');
const app = express();
const PORT = process.env.PORT || 3000;
// Storage manager routes
const storageManager = require('./storage-manager');
app.use('/api/clones', storageManager);
// Security headers
app.use((req, res, next) => {
res.setHeader('X-Content-Type-Options', 'nosniff');
res.setHeader('X-Frame-Options', 'DENY');
res.setHeader('X-XSS-Protection', '1; mode=block');
res.setHeader('Referrer-Policy', 'strict-origin-when-cross-origin');
next();
});
// Middleware
app.use(cors());
app.use(express.json());
// Helper para parsear cookies
function parseCookies(cookieHeader) {
const list = {};
if (!cookieHeader) return list;
cookieHeader.split(';').forEach(cookie => {
let [name, ...rest] = cookie.split('=');
name = name?.trim();
if (!name) return;
const value = rest.join('=').trim();
if (!value) return;
list[name] = decodeURIComponent(value);
});
return list;
}
// Rota para ativar o Virtual Root Preview (define o cookie e redireciona para a raiz)
app.get('/api/preview/:folder', (req, res) => {
const folderName = req.params.folder;
res.setHeader('Set-Cookie', `clone_preview_target=${encodeURIComponent(folderName)}; Path=/; HttpOnly; SameSite=Lax`);
res.redirect('/');
});
// Rota para sair do Virtual Root Preview (limpa o cookie e volta ao dashboard)
app.get('/api/exit-preview', (req, res) => {
res.setHeader('Set-Cookie', 'clone_preview_target=; Path=/; Expires=Thu, 01 Jan 1970 00:00:00 GMT');
res.redirect('/');
});
// Middleware de Virtual Root Proxy (serve o site clonado diretamente na raiz / quando o cookie está ativo)
app.use((req, res, next) => {
const cookies = parseCookies(req.headers.cookie);
if (cookies.clone_preview_target && !req.path.startsWith('/api') && req.path !== '/old') {
const folderName = cookies.clone_preview_target;
const folderPath = path.join(__dirname, '..', 'cloned-sites', folderName);
if (fs.existsSync(folderPath)) {
let targetFile = req.path;
if (targetFile === '/') targetFile = '/index.html';
// Se o arquivo solicitado existir dentro da pasta do clone
const fullPath = path.join(folderPath, targetFile);
if (fs.existsSync(fullPath) && fs.statSync(fullPath).isFile()) {
// Se for HTML, injeta a barra de preview flutuante
if (targetFile.endsWith('.html')) {
let html = fs.readFileSync(fullPath, 'utf8');
const previewBar = `
`;
if (html.includes('')) {
html = html.replace('', previewBar + '');
} else {
html += previewBar;
}
res.setHeader('Content-Type', 'text/html');
return res.send(html);
}
// Para outros arquivos (CSS, JS, imagens), serve diretamente com o MIME type correto
if (targetFile.endsWith('.css')) res.setHeader('Content-Type', 'text/css');
if (targetFile.endsWith('.js')) res.setHeader('Content-Type', 'application/javascript');
return res.sendFile(fullPath);
} else {
// Fallback de SPA: se a rota virtual não existir como arquivo no disco, serve o index.html do clone!!!
const indexPath = path.join(folderPath, 'index.html');
if (fs.existsSync(indexPath)) {
let html = fs.readFileSync(indexPath, 'utf8');
const previewBar = `
`;
if (html.includes('')) html = html.replace('', previewBar + '');
else html += previewBar;
res.setHeader('Content-Type', 'text/html');
return res.send(html);
}
}
}
}
next();
});
// Serve static files with proper MIME types
app.use(express.static(path.join(__dirname, 'public'), {
setHeaders: (res, path) => {
if (path.endsWith('.css')) res.setHeader('Content-Type', 'text/css');
if (path.endsWith('.js')) res.setHeader('Content-Type', 'application/javascript');
if (path.endsWith('.html')) res.setHeader('Content-Type', 'text/html');
}
}));
// Middleware para injetar fix de roteador em sites clonados (SPAs como React, Vue, Angular, Vite) e reescrever JS
app.use((req, res, next) => {
if (!req.path.startsWith('/api') && req.path !== '/' && req.path !== '/old') {
let urlPath = req.path;
// Remover prefixo /cloned-sites se existir para buscar corretamente no disco
if (urlPath.startsWith('/cloned-sites/')) {
urlPath = urlPath.replace('/cloned-sites/', '/');
}
if (!urlPath.match(/\.[a-zA-Z0-9]+$/)) {
if (!urlPath.endsWith('/')) urlPath += '/';
urlPath += 'index.html';
}
// Interceptação de arquivos HTML (Camada 1 - DOM Mock Universal)
if (urlPath.endsWith('.html')) {
const filePath = path.join(__dirname, '..', 'cloned-sites', urlPath);
if (fs.existsSync(filePath)) {
try {
let html = fs.readFileSync(filePath, 'utf8');
if (!html.includes('__CLONEWEB_ROUTER_FIX__')) {
const injectScript = `
`;
if (html.includes(' ')) {
html = html.replace('
' + injectScript);
} else {
html = injectScript + html;
}
}
res.setHeader('Content-Type', 'text/html');
return res.send(html);
} catch (err) {
console.error('Erro ao injetar fix no HTML do clone:', err);
}
}
}
// Interceptação de arquivos JS de sites clonados (Camada 2 - JS Rewriter Supremo)
if (urlPath.endsWith('.js') && (urlPath.includes('_202') || urlPath.includes('/cloned-sites/'))) {
const filePath = path.join(__dirname, '..', 'cloned-sites', urlPath);
if (fs.existsSync(filePath)) {
try {
let jsContent = fs.readFileSync(filePath, 'utf8');
// Substitui window.location.pathname de forma segura
jsContent = jsContent.replace(/\bwindow\.location\.pathname\b/g, '(window.__mockLocationPathname || window.location.pathname)');
// Substitui location.pathname isolado (evita quebrar e.location.pathname ou history.location.pathname)
jsContent = jsContent.replace(/(? {
if (filePath.endsWith('.css')) res.setHeader('Content-Type', 'text/css');
if (filePath.endsWith('.js')) res.setHeader('Content-Type', 'application/javascript');
}
}));
// Proxy para o crawler service - SEM TIMEOUT
app.post('/api/crawl', async (req, res) => {
try {
console.log('🚀 Proxying crawl request to crawler service...');
console.log('⏳ Clonagem pode demorar vários minutos para sites grandes...');
// SEM TIMEOUT - permite clonagens longas
const response = await axios.post('http://localhost:3001/crawl', req.body, {
timeout: 0 // Sem limite de tempo
});
console.log('✅ Crawl completed successfully');
res.json(response.data);
} catch (error) {
console.error('❌ Crawl error:', error.message);
res.status(500).json({
success: false,
error: error.response?.data?.error || error.message
});
}
});
// Proxy para o wget cloner - SEM TIMEOUT para clonagens grandes
app.post('/api/wget-clone', async (req, res) => {
try {
console.log('🔧 Proxying wget clone request...');
console.log('⏳ Clonagem pode demorar vários minutos para sites grandes...');
// SEM TIMEOUT - permite clonagens que demoram mais de 10 minutos
const response = await axios.post('http://localhost:3002/wget-clone', req.body, {
timeout: 0 // Sem limite de tempo
});
console.log('✅ Wget clone completed successfully');
// Sync to host /root/Desktop/Clonados
const directory = response.data.data?.directory || response.data.directory;
if (response.data && response.data.success && directory) {
const { exec } = require('child_process');
const containerId = require('os').hostname();
const cloneDir = directory.split('/').pop();
const hostDir = req.body.outputDir || '/root/Desktop/Clonados';
exec(`mkdir -p ${hostDir} && cp -r ${directory} ${hostDir}/ 2>/dev/null || docker cp ${containerId}:${directory} ${hostDir}/ 2>/dev/null`, (err) => {
if (err) console.log('⚠️ Sync to host failed:', err.message);
else console.log('✅ Clone synced to host Desktop:', cloneDir);
});
}
res.json(response.data);
} catch (error) {
console.error('❌ Wget clone error:', error.message);
res.status(500).json({
success: false,
error: error.response?.data?.error || error.message
});
}
});
// Smart Clone Endpoint (Local Spawn)
app.post('/api/smart-clone', (req, res) => {
const { url } = req.body;
if (!url) {
return res.status(400).json({ error: 'URL is required' });
}
console.log('🚀 Starting Smart Clone for:', url);
console.log('⏳ This uses Puppeteer and may take a while...');
// Spawn the smart cloner process
const scriptPath = path.join(__dirname, '..', 'simple-crawler', 'smart-cloner.js');
console.log(`[SmartClone] Executing: node "${scriptPath}" --url="${url}"`);
const child = spawn('node', [scriptPath, `--url=${url}`], {
cwd: path.join(__dirname, '..'),
shell: true
});
let output = '';
let errorOutput = '';
child.stdout.on('data', (data) => {
const text = data.toString();
output += text;
console.log(`[SmartClone STDOUT] ${text.trim()}`);
});
child.stderr.on('data', (data) => {
const text = data.toString();
errorOutput += text;
console.error(`[SmartClone STDERR] ${text.trim()}`);
});
child.on('error', (err) => {
console.error('[SmartClone SPAWN ERROR]', err);
if (!res.headersSent) {
res.status(500).json({ success: false, error: 'Failed to start cloning process', details: err.message });
}
});
child.on('close', (code) => {
console.log(`[SmartClone] Process exited with code ${code}`);
if (res.headersSent) return;
try {
// Find the last JSON object in output
const matches = output.match(/\{[\s\S]*\}/g);
if (matches && matches.length > 0) {
const lastMatch = matches[matches.length - 1];
const result = JSON.parse(lastMatch);
// Sync to host /root/Desktop/Clonados
const directory = result.data?.directory || result.directory || result.path;
if (result.success && directory) {
const { exec } = require('child_process');
const containerId = require('os').hostname();
const cloneDir = directory.split('/').pop();
const hostDir = req.body.outputDir || '/root/Desktop/Clonados';
exec(`mkdir -p ${hostDir} && cp -r ${directory} ${hostDir}/ 2>/dev/null || docker cp ${containerId}:${directory} ${hostDir}/ 2>/dev/null`, (err) => {
if (err) console.log('⚠️ SmartClone sync to host failed:', err.message);
else console.log('✅ SmartClone synced to host Desktop:', cloneDir);
});
}
res.json({
success: result.success,
data: result,
error: result.error
});
} else {
throw new Error('No valid JSON output found from cloner');
}
} catch (e) {
console.error('[SmartClone PARSE ERROR]', e.message);
if (code === 0) {
res.json({ success: true, message: 'Process finished but output parsing failed', fullOutput: output });
} else {
res.status(500).json({ success: false, error: 'Clone failed or output was invalid', details: errorOutput });
}
}
});
});
// Health check do wget cloner (porta 3002)
app.get('/api/health', async (req, res) => {
try {
const response = await axios.get('http://localhost:3002/health', {
timeout: 5000
});
res.json(response.data);
} catch (error) {
res.status(500).json({
status: 'error',
message: 'WGET Cloner service unavailable'
});
}
});
// Rota de visualização de arquivos da pasta clonada (Gerenciador de Arquivos Web Embutido)
app.get('/api/browse/:folder', (req, res) => {
try {
const folderName = req.params.folder;
const folderPath = path.join(__dirname, '..', 'cloned-sites', folderName);
if (!fs.existsSync(folderPath)) {
return res.status(404).send('Pasta não encontrada A pasta solicitada não existe no servidor.
');
}
const items = fs.readdirSync(folderPath);
let filesHtml = '';
items.forEach(item => {
const itemPath = path.join(folderPath, item);
const stat = fs.statSync(itemPath);
const isDir = stat.isDirectory();
const size = isDir ? '-' : (stat.size / 1024).toFixed(2) + ' KB';
const date = new Date(stat.mtime).toLocaleString('pt-BR');
const icon = isDir ? 'fa-folder' : (item.endsWith('.html') ? 'fa-file-code' : (item.endsWith('.json') ? 'fa-file-alt' : 'fa-file'));
const link = isDir ? `#` : `/${folderName}/${item}`;
filesHtml += `
${item} ${size}
${date}
${!isDir ? `Visualizar ` : ''}
${!isDir ? `Baixar ` : ''}
`;
});
const html = `
Arquivos Clonados - ${folderName}
Nome do Ficheiro
Tamanho
Modificado em
Ações
${filesHtml}