chore: update dependencies and configs
This commit is contained in:
@@ -9,9 +9,11 @@
|
||||
"test": "echo \"Test with curl command\""
|
||||
},
|
||||
"dependencies": {
|
||||
"axios": "^1.13.2",
|
||||
"axios": "^1.16.1",
|
||||
"cheerio": "^1.0.0-rc.12",
|
||||
"express": "^4.18.2",
|
||||
"puppeteer": "^21.5.2"
|
||||
"playwright": "^1.60.0",
|
||||
"playwright-extra": "^4.3.6",
|
||||
"puppeteer-extra-plugin-stealth": "^2.11.2"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+122
-12
@@ -1,7 +1,8 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const puppeteer = require('puppeteer');
|
||||
const { chromium } = require('playwright');
|
||||
const { URL } = require('url');
|
||||
const aiService = require('./ai-recreation');
|
||||
|
||||
class SmartCloner {
|
||||
constructor() {
|
||||
@@ -84,14 +85,15 @@ class SmartCloner {
|
||||
const siteDir = path.join(this.baseOutputDir, `${siteName}_${timestamp}`);
|
||||
this.ensureDirectoryExists(siteDir);
|
||||
|
||||
browser = await puppeteer.launch({
|
||||
headless: 'new',
|
||||
browser = await chromium.launch({
|
||||
headless: true,
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox', '--window-size=1920,1080']
|
||||
});
|
||||
|
||||
const pagesToVisit = [url];
|
||||
const visitedPages = new Set();
|
||||
const pageMap = new Map(); // URL -> LocalFilename
|
||||
const apiMocks = []; // Array of { url, method, status, headers, body }
|
||||
|
||||
// Pre-seed the map so links can be rewritten to files we haven't visited yet
|
||||
pageMap.set(url.replace(/\/$/, ''), 'index.html');
|
||||
@@ -108,16 +110,36 @@ class SmartCloner {
|
||||
|
||||
console.log(`📄 [${pageCount}/${maxPages}] Clonando: ${currentUrl}`);
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.setViewport({ width: 1920, height: 1080 });
|
||||
const context = await browser.newContext({ viewport: { width: 1920, height: 1080 }});
|
||||
const page = await context.newPage();
|
||||
|
||||
// Asset interception
|
||||
// Asset and API interception
|
||||
page.on('response', async (response) => {
|
||||
const rUrl = response.url();
|
||||
if (rUrl.startsWith('data:') || !response.ok() || response.request().method() !== 'GET') return;
|
||||
const req = response.request();
|
||||
|
||||
if (rUrl.startsWith('data:') || !response.ok()) return;
|
||||
|
||||
const resourceType = req.resourceType();
|
||||
if (resourceType === 'fetch' || resourceType === 'xhr') {
|
||||
try {
|
||||
const bodyStr = await response.text();
|
||||
apiMocks.push({
|
||||
url: rUrl,
|
||||
method: req.method(),
|
||||
status: response.status(),
|
||||
headers: response.headers(),
|
||||
body: bodyStr
|
||||
});
|
||||
} catch (e) { }
|
||||
return;
|
||||
}
|
||||
|
||||
if (req.method() !== 'GET') return;
|
||||
try {
|
||||
const contentType = response.headers()['content-type'];
|
||||
const buffer = await response.buffer();
|
||||
const headers = response.headers();
|
||||
const contentType = headers['content-type'];
|
||||
const buffer = await response.body();
|
||||
const localPath = this.generateLocalPath(rUrl, contentType, siteDir);
|
||||
this.ensureDirectoryExists(path.dirname(localPath));
|
||||
fs.writeFileSync(localPath, buffer);
|
||||
@@ -126,10 +148,10 @@ class SmartCloner {
|
||||
});
|
||||
|
||||
try {
|
||||
await page.goto(currentUrl, { waitUntil: 'networkidle2', timeout: 60000 });
|
||||
await page.goto(currentUrl, { waitUntil: 'networkidle', timeout: 60000 });
|
||||
} catch (e) {
|
||||
console.log(`⚠️ Falha ao carregar ${currentUrl}: ${e.message}`);
|
||||
await page.close();
|
||||
await context.close();
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -212,16 +234,104 @@ class SmartCloner {
|
||||
})();
|
||||
`;
|
||||
document.body.appendChild(script);
|
||||
|
||||
// 4. Inject Service Worker Registration
|
||||
const swScript = document.createElement('script');
|
||||
swScript.textContent = `
|
||||
if ('serviceWorker' in navigator) {
|
||||
window.addEventListener('load', function() {
|
||||
navigator.serviceWorker.register('/mock-sw.js').then(function(registration) {
|
||||
console.log('Mock ServiceWorker registration successful with scope: ', registration.scope);
|
||||
}, function(err) {
|
||||
console.log('Mock ServiceWorker registration failed: ', err);
|
||||
});
|
||||
});
|
||||
}
|
||||
`;
|
||||
document.body.appendChild(swScript);
|
||||
}, resMapArr, pagMapArr);
|
||||
|
||||
// 5. AI Recreation of forms and dynamic content
|
||||
const dynamicElements = await page.evaluate(() => {
|
||||
return Array.from(document.querySelectorAll('form, [data-dynamic], .dynamic-content')).map(f => ({
|
||||
html: f.outerHTML,
|
||||
tagName: f.tagName
|
||||
}));
|
||||
});
|
||||
|
||||
for (const el of dynamicElements) {
|
||||
const newHtml = await aiService.recreateComponent(el.html, el.tagName);
|
||||
if (newHtml) {
|
||||
await page.evaluate(({newHtml, oldHtml}) => {
|
||||
const nodes = Array.from(document.querySelectorAll('form, [data-dynamic], .dynamic-content'));
|
||||
const target = nodes.find(n => n.outerHTML === oldHtml);
|
||||
if (target) {
|
||||
target.outerHTML = newHtml;
|
||||
}
|
||||
}, { newHtml, oldHtml: el.html });
|
||||
}
|
||||
}
|
||||
|
||||
// 6. Expand Shadow DOM to Declarative Shadow DOM
|
||||
await page.evaluate(() => {
|
||||
function expandShadowRoots(node) {
|
||||
if (node.shadowRoot) {
|
||||
const template = document.createElement('template');
|
||||
template.setAttribute('shadowrootmode', 'open');
|
||||
template.innerHTML = node.shadowRoot.innerHTML;
|
||||
node.insertBefore(template, node.firstChild);
|
||||
// Recursively process inside shadowRoot
|
||||
Array.from(node.shadowRoot.querySelectorAll('*')).forEach(expandShadowRoots);
|
||||
}
|
||||
Array.from(node.children).forEach(expandShadowRoots);
|
||||
}
|
||||
expandShadowRoots(document.body);
|
||||
});
|
||||
|
||||
const html = await page.content();
|
||||
const localFile = pageMap.get(cleanCurrentUrl) || 'index.html';
|
||||
fs.writeFileSync(path.join(siteDir, localFile), html);
|
||||
await page.close();
|
||||
await context.close();
|
||||
}
|
||||
|
||||
const info = { url, timestamp: new Date().toISOString(), pages: pageCount, directory: siteDir };
|
||||
fs.writeFileSync(path.join(siteDir, 'clone-info.json'), JSON.stringify(info, null, 2));
|
||||
|
||||
// Save Mock DB
|
||||
fs.writeFileSync(path.join(siteDir, 'mock-db.json'), JSON.stringify(apiMocks, null, 2));
|
||||
|
||||
// Create Service Worker
|
||||
const swCode = `
|
||||
const MOCK_DB_URL = '/mock-db.json';
|
||||
let mockDb = null;
|
||||
|
||||
self.addEventListener('install', (event) => {
|
||||
self.skipWaiting();
|
||||
});
|
||||
|
||||
self.addEventListener('activate', (event) => {
|
||||
event.waitUntil(clients.claim());
|
||||
event.waitUntil(
|
||||
fetch(MOCK_DB_URL).then(res => res.json()).then(data => { mockDb = data; })
|
||||
);
|
||||
});
|
||||
|
||||
self.addEventListener('fetch', (event) => {
|
||||
if (event.request.method === 'GET' && !event.request.url.includes('/mock-db.json') && mockDb) {
|
||||
const mock = mockDb.find(m => m.url === event.request.url || event.request.url.includes(m.url));
|
||||
if (mock) {
|
||||
event.respondWith(
|
||||
new Response(mock.body, {
|
||||
status: mock.status,
|
||||
headers: { 'Content-Type': mock.headers['content-type'] || 'application/json' }
|
||||
})
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
});
|
||||
`;
|
||||
fs.writeFileSync(path.join(siteDir, 'mock-sw.js'), swCode);
|
||||
|
||||
return { success: true, path: siteDir, pagesCloned: pageCount };
|
||||
} catch (e) {
|
||||
|
||||
Reference in New Issue
Block a user