const express = require('express');
const path = require('path');
const cors = require('cors');
const axios = require('axios');
const { exec, spawn } = require('child_process');
const fs = require('fs');
const app = express();
const PORT = process.env.PORT || 3000;
// Storage manager routes
const storageManager = require('./storage-manager');
app.use('/api/clones', storageManager);
// Security headers
app.use((req, res, next) => {
res.setHeader('X-Content-Type-Options', 'nosniff');
res.setHeader('X-Frame-Options', 'DENY');
res.setHeader('X-XSS-Protection', '1; mode=block');
res.setHeader('Referrer-Policy', 'strict-origin-when-cross-origin');
next();
});
// Middleware
app.use(cors());
app.use(express.json());
// Serve static files with proper MIME types
app.use(express.static(path.join(__dirname, 'public'), {
setHeaders: (res, path) => {
if (path.endsWith('.css')) res.setHeader('Content-Type', 'text/css');
if (path.endsWith('.js')) res.setHeader('Content-Type', 'application/javascript');
if (path.endsWith('.html')) res.setHeader('Content-Type', 'text/html');
}
}));
// Middleware para injetar fix de roteador em sites clonados (SPAs como React, Vue, Angular, Vite)
app.use((req, res, next) => {
if (!req.path.startsWith('/api') && req.path !== '/' && req.path !== '/old') {
let urlPath = req.path;
if (!urlPath.match(/\.[a-zA-Z0-9]+$/)) {
if (!urlPath.endsWith('/')) urlPath += '/';
urlPath += 'index.html';
}
if (urlPath.endsWith('.html')) {
const filePath = path.join(__dirname, '..', 'cloned-sites', urlPath);
if (fs.existsSync(filePath)) {
try {
let html = fs.readFileSync(filePath, 'utf8');
if (!html.includes('__CLONEWEB_ROUTER_FIX__')) {
const injectScript = `
`;
if (html.includes('')) {
html = html.replace('', '' + injectScript);
} else {
html = injectScript + html;
}
}
res.setHeader('Content-Type', 'text/html');
return res.send(html);
} catch (err) {
console.error('Erro ao injetar fix no HTML do clone:', err);
}
}
}
}
next();
});
// Serve cloned sites directory
app.use(express.static(path.join(__dirname, '..', 'cloned-sites'), {
setHeaders: (res, filePath) => {
if (filePath.endsWith('.css')) res.setHeader('Content-Type', 'text/css');
if (filePath.endsWith('.js')) res.setHeader('Content-Type', 'application/javascript');
}
}));
// Proxy para o crawler service - SEM TIMEOUT
app.post('/api/crawl', async (req, res) => {
try {
console.log('🚀 Proxying crawl request to crawler service...');
console.log('⏳ Clonagem pode demorar vários minutos para sites grandes...');
// SEM TIMEOUT - permite clonagens longas
const response = await axios.post('http://localhost:3001/crawl', req.body, {
timeout: 0 // Sem limite de tempo
});
console.log('✅ Crawl completed successfully');
res.json(response.data);
} catch (error) {
console.error('❌ Crawl error:', error.message);
res.status(500).json({
success: false,
error: error.response?.data?.error || error.message
});
}
});
// Proxy para o wget cloner - SEM TIMEOUT para clonagens grandes
app.post('/api/wget-clone', async (req, res) => {
try {
console.log('🔧 Proxying wget clone request...');
console.log('⏳ Clonagem pode demorar vários minutos para sites grandes...');
// SEM TIMEOUT - permite clonagens que demoram mais de 10 minutos
const response = await axios.post('http://localhost:3002/wget-clone', req.body, {
timeout: 0 // Sem limite de tempo
});
console.log('✅ Wget clone completed successfully');
// Sync to host /root/Desktop/Clonados
const directory = response.data.data?.directory || response.data.directory;
if (response.data && response.data.success && directory) {
const { exec } = require('child_process');
const containerId = require('os').hostname();
const cloneDir = directory.split('/').pop();
const hostDir = '/root/Desktop/Clonados';
exec(`mkdir -p ${hostDir} && cp -r ${directory} ${hostDir}/ 2>/dev/null || docker cp ${containerId}:${directory} ${hostDir}/ 2>/dev/null`, (err) => {
if (err) console.log('⚠️ Sync to host failed:', err.message);
else console.log('✅ Clone synced to host Desktop:', cloneDir);
});
}
res.json(response.data);
} catch (error) {
console.error('❌ Wget clone error:', error.message);
res.status(500).json({
success: false,
error: error.response?.data?.error || error.message
});
}
});
// Smart Clone Endpoint (Local Spawn)
app.post('/api/smart-clone', (req, res) => {
const { url } = req.body;
if (!url) {
return res.status(400).json({ error: 'URL is required' });
}
console.log('🚀 Starting Smart Clone for:', url);
console.log('⏳ This uses Puppeteer and may take a while...');
// Spawn the smart cloner process
const scriptPath = path.join(__dirname, '..', 'simple-crawler', 'smart-cloner.js');
console.log(`[SmartClone] Executing: node "${scriptPath}" --url="${url}"`);
const child = spawn('node', [scriptPath, `--url=${url}`], {
cwd: path.join(__dirname, '..'),
shell: true
});
let output = '';
let errorOutput = '';
child.stdout.on('data', (data) => {
const text = data.toString();
output += text;
console.log(`[SmartClone STDOUT] ${text.trim()}`);
});
child.stderr.on('data', (data) => {
const text = data.toString();
errorOutput += text;
console.error(`[SmartClone STDERR] ${text.trim()}`);
});
child.on('error', (err) => {
console.error('[SmartClone SPAWN ERROR]', err);
if (!res.headersSent) {
res.status(500).json({ success: false, error: 'Failed to start cloning process', details: err.message });
}
});
child.on('close', (code) => {
console.log(`[SmartClone] Process exited with code ${code}`);
if (res.headersSent) return;
try {
// Find the last JSON object in output
const matches = output.match(/\{[\s\S]*\}/g);
if (matches && matches.length > 0) {
const lastMatch = matches[matches.length - 1];
const result = JSON.parse(lastMatch);
// Sync to host /root/Desktop/Clonados
const directory = result.data?.directory || result.directory || result.path;
if (result.success && directory) {
const { exec } = require('child_process');
const containerId = require('os').hostname();
const cloneDir = directory.split('/').pop();
const hostDir = '/root/Desktop/Clonados';
exec(`mkdir -p ${hostDir} && cp -r ${directory} ${hostDir}/ 2>/dev/null || docker cp ${containerId}:${directory} ${hostDir}/ 2>/dev/null`, (err) => {
if (err) console.log('⚠️ SmartClone sync to host failed:', err.message);
else console.log('✅ SmartClone synced to host Desktop:', cloneDir);
});
}
res.json({
success: result.success,
data: result,
error: result.error
});
} else {
throw new Error('No valid JSON output found from cloner');
}
} catch (e) {
console.error('[SmartClone PARSE ERROR]', e.message);
if (code === 0) {
res.json({ success: true, message: 'Process finished but output parsing failed', fullOutput: output });
} else {
res.status(500).json({ success: false, error: 'Clone failed or output was invalid', details: errorOutput });
}
}
});
});
// Health check do wget cloner (porta 3002)
app.get('/api/health', async (req, res) => {
try {
const response = await axios.get('http://localhost:3002/health', {
timeout: 5000
});
res.json(response.data);
} catch (error) {
res.status(500).json({
status: 'error',
message: 'WGET Cloner service unavailable'
});
}
});
// Abrir pasta do clone
app.post('/api/open-folder', (req, res) => {
try {
const { path: folderPath } = req.body;
if (!folderPath) {
return res.status(400).json({ error: 'Caminho da pasta é obrigatório' });
}
// Abrir pasta no gerenciador de ficheiros
exec(`xdg-open "${folderPath}"`, (error) => {
if (error) {
console.error('Erro ao abrir pasta:', error);
return res.status(500).json({ error: 'Não foi possível abrir a pasta' });
}
res.json({ success: true, message: 'Pasta aberta com sucesso' });
});
} catch (error) {
console.error('Erro ao abrir pasta:', error);
res.status(500).json({ error: 'Erro interno do servidor' });
}
});
// Abrir site clonado via HTTP para evitar problemas de origem/CORS
app.post('/api/open-site', (req, res) => {
try {
const { path: sitePath } = req.body;
if (!sitePath) {
return res.status(400).json({ error: 'Caminho do site é obrigatório' });
}
// Extrair apenas o nome da pasta (último segmento do caminho)
const folderName = path.basename(sitePath);
const siteUrl = `http://localhost:${PORT}/${folderName}/index.html`;
console.log(`🌍 Opening site via HTTP: ${siteUrl}`);
// Abrir URL no navegador padrão (Linux)
exec(`xdg-open "${siteUrl}"`, (error) => {
if (error) {
console.error('Erro ao abrir site:', error);
return res.status(500).json({ error: 'Não foi possível abrir o site via HTTP' });
}
res.json({ success: true, message: 'Site aberto via HTTP com sucesso', url: siteUrl });
});
} catch (error) {
console.error('Erro ao abrir site:', error);
res.status(500).json({ error: 'Erro interno do servidor' });
}
});
// Abrir página de navegação via HTTP
app.post('/api/open-navigation', (req, res) => {
try {
const { path: sitePath } = req.body;
if (!sitePath) {
return res.status(400).json({ error: 'Caminho do site é obrigatório' });
}
const folderName = path.basename(sitePath);
const navUrl = `http://localhost:${PORT}/${folderName}/navegacao.html`;
console.log(`🌍 Opening navigation via HTTP: ${navUrl}`);
// Abrir navegacao.html no navegador padrão
exec(`start "" "${navUrl}"`, (error) => {
if (error) {
console.error('Erro ao abrir navegação:', error);
return res.status(500).json({ error: 'Não foi possível abrir a navegação via HTTP' });
}
res.json({ success: true, message: 'Navegação aberta via HTTP com sucesso', url: navUrl });
});
} catch (error) {
console.error('Erro ao abrir navegação:', error);
res.status(500).json({ error: 'Erro interno do servidor' });
}
});
app.get('/', (req, res) => {
res.sendFile(path.join(__dirname, 'public', 'index-new.html'));
});
// Serve old interface
app.get('/old', (req, res) => {
res.sendFile(path.join(__dirname, 'public', 'index.html'));
});
// Serve favicon requests
app.get('/favicon.ico', (req, res) => {
res.redirect('/favicon.svg');
});
// Handle common missing resources
app.get('/index', (req, res) => {
res.redirect('/');
});
// AI Agent Route (MiniMax)
app.post('/api/ai/agent', async (req, res) => {
const { systemPrompt, userMessage } = req.body;
const apiKey = process.env.MINIMAX_API_KEY;
if (!apiKey) {
return res.status(500).json({ error: 'MINIMAX_API_KEY is missing in .env' });
}
try {
const response = await axios.post('https://api.minimax.io/v1/chat/completions', {
model: "MiniMax-M2.7",
messages: [
{ role: "system", content: systemPrompt || "Você é um assistente de inteligência artificial útil." },
{ role: "user", content: userMessage }
],
max_tokens: 2048
}, {
headers: {
'Authorization': `Bearer ${apiKey}`,
'Content-Type': 'application/json'
},
timeout: 90000 // 90s
});
let reply = response.data.choices?.[0]?.message?.content || 'Processamento concluído, porém sem retorno.';
// Strip reasoning tags if present
reply = reply.replace(/[\s\S]*?<\/think>/gi, '').trim();
res.json({ success: true, data: reply });
} catch (error) {
console.error('MiniMax AI Error:', error.response?.data || error.message);
res.status(500).json({ error: error.message, details: error.response?.data });
}
});
// Handle 404s
app.use((req, res) => {
res.status(404).json({
error: 'Not Found',
message: `Route ${req.path} not found`
});
});
// Error handler
app.use((err, req, res, next) => {
console.error('Server error:', err);
res.status(500).json({
error: 'Internal Server Error',
message: err.message
});
});
app.listen(PORT, () => {
console.log(`🌐 CloneWeb Interface running on http://localhost:${PORT}`);
console.log(`🔗 Make sure crawler service is running on http://localhost:3001`);
console.log(`📁 Serving static files from: ${path.join(__dirname, 'public')}`);
});