admtracksteel/webclone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: update dependencies and configs

Browse Source
This commit is contained in:
admtracksteel
2026-05-17 19:23:16 +00:00
parent 9cba552453
commit 3628719b3b
8 changed files with 881 additions and 1040 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package-lock.json
Generated
+709 -983
View File
File diff suppressed because it is too large Load Diff
packages/database/package.json
+3 -3
View File
@@ -19,8 +19,8 @@
"db:reset": "prisma db push --force-reset"
},
"dependencies": {
"@prisma/client": "^5.4.2",
"@cloneweb/shared": "file:../shared",
"@prisma/client": "^5.4.2",
"prisma": "^5.4.2"
},
"devDependencies": {
@@ -28,7 +28,7 @@
"fast-check": "^3.13.2",
"jest": "^29.7.0",
"ts-jest": "^29.1.1",
"tsx": "^3.14.0",
"tsx": "^4.22.1",
"typescript": "^5.2.2"
}
}
}
services/api-gateway/package.json
+13 -13
View File
@@ -13,31 +13,31 @@
"test:watch": "jest --watch"
},
"dependencies": {
"@cloneweb/types": "file:../../packages/types",
"@cloneweb/database": "file:../../packages/database",
"@cloneweb/shared": "file:../../packages/shared",
"express": "^4.18.2",
"cors": "^2.8.5",
"helmet": "^7.1.0",
"express-rate-limit": "^7.1.5",
"jsonwebtoken": "^9.0.2",
"@cloneweb/types": "file:../../packages/types",
"bcryptjs": "^2.4.3",
"zod": "^3.22.4",
"cors": "^2.8.5",
"dotenv": "^16.3.1",
"winston": "^3.11.0"
"express": "^4.18.2",
"express-rate-limit": "^7.1.5",
"helmet": "^7.1.0",
"jsonwebtoken": "^9.0.2",
"winston": "^3.11.0",
"zod": "^3.22.4"
},
"devDependencies": {
"@types/express": "^4.17.21",
"@types/cors": "^2.8.17",
"@types/jsonwebtoken": "^9.0.5",
"@types/bcryptjs": "^2.4.6",
"@types/cors": "^2.8.17",
"@types/express": "^4.17.21",
"@types/jest": "^29.5.5",
"@types/jsonwebtoken": "^9.0.5",
"@types/supertest": "^2.0.16",
"fast-check": "^3.13.2",
"jest": "^29.7.0",
"supertest": "^6.3.3",
"ts-jest": "^29.1.1",
"tsx": "^3.14.0",
"tsx": "^4.22.1",
"typescript": "^5.2.2"
}
}
}
services/api-gateway/tsconfig.tsbuildinfo
+1 -1
View File
File diff suppressed because one or more lines are too long
services/crawler/package.json
+14 -12
View File
@@ -11,27 +11,29 @@
"test:watch": "jest --watch"
},
"dependencies": {
"@bull-board/api": "^5.10.2",
"@bull-board/express": "^5.10.2",
"@cloneweb/database": "file:../../packages/database",
"@cloneweb/shared": "file:../../packages/shared",
"@cloneweb/types": "file:../../packages/types",
"express": "^4.18.2",
"puppeteer": "^21.5.2",
"bull": "^4.12.2",
"@bull-board/api": "^5.10.2",
"@bull-board/express": "^5.10.2",
"ioredis": "^5.3.2",
"robots-parser": "^3.0.1",
"url-parse": "^1.5.10",
"cheerio": "^1.0.0-rc.12",
"fast-check": "^3.15.0"
"express": "^4.18.2",
"fast-check": "^3.15.0",
"ioredis": "^5.3.2",
"playwright": "^1.60.0",
"playwright-extra": "^4.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2",
"robots-parser": "^3.0.1",
"url-parse": "^1.5.10"
},
"devDependencies": {
"@types/express": "^4.17.21",
"@types/jest": "^29.5.8",
"@types/node": "^20.10.4",
"@types/url-parse": "^1.4.11",
"typescript": "^5.3.3",
"ts-node-dev": "^2.0.0",
"jest": "^29.7.0",
"@types/jest": "^29.5.8"
"ts-node-dev": "^2.0.0",
"typescript": "^5.3.3"
}
}
}
services/crawler/src/core/IntelligentCrawler.ts
+14 -13
View File
@@ -1,4 +1,4 @@
import puppeteer, { Browser, Page } from 'puppeteer';
import { chromium, Browser, Page, BrowserContext } from 'playwright';
import { URL } from 'url';
import * as cheerio from 'cheerio';
import robotsParser from 'robots-parser';
@@ -13,8 +13,8 @@ export class IntelligentCrawler {
private lastRequestTime = 0;
async initialize(): Promise<void> {
this.browser = await puppeteer.launch({
headless: 'new',
this.browser = await chromium.launch({
headless: true,
args: [
'--no-sandbox',
'--disable-setuid-sandbox',
@@ -26,7 +26,7 @@ export class IntelligentCrawler {
'--disable-gpu'
]
});
logger.info('Puppeteer browser initialized');
logger.info('Playwright browser initialized');
}
async crawlWebsite(startUrl: string, config: CrawlerConfig): Promise<SiteMap> {
@@ -89,18 +89,18 @@ export class IntelligentCrawler {
}
async extractPageInfo(url: string, depth: number, config: CrawlerConfig): Promise<PageInfo | null> {
const page = await this.browser!.newPage();
const context = await this.browser!.newContext({
viewport: config.viewport,
userAgent: config.userAgent
});
const page = await context.newPage();
try {
// Set viewport and user agent
await page.setViewport(config.viewport);
await page.setUserAgent(config.userAgent);
const startTime = Date.now();
// Navigate to page
const response = await page.goto(url, {
waitUntil: 'networkidle2',
waitUntil: 'networkidle',
timeout: config.timeout
});
@@ -162,7 +162,7 @@ export class IntelligentCrawler {
};
} finally {
await page.close();
await context.close();
}
}
@@ -224,7 +224,8 @@ export class IntelligentCrawler {
try {
const robotsUrl = `${origin}/robots.txt`;
const page = await this.browser!.newPage();
const context = await this.browser!.newContext();
const page = await context.newPage();
const response = await page.goto(robotsUrl, { timeout: 10000 });
@@ -235,7 +236,7 @@ export class IntelligentCrawler {
logger.info(`Loaded robots.txt for ${origin}`);
}
await page.close();
await context.close();
} catch (error) {
logger.warn(`Failed to load robots.txt for ${origin}:`, error);
this.robotsCache.set(origin, null);
simple-crawler/package.json
+5 -3
View File
@@ -9,9 +9,11 @@
"test": "echo \"Test with curl command\""
},
"dependencies": {
"axios": "^1.13.2",
"axios": "^1.16.1",
"cheerio": "^1.0.0-rc.12",
"express": "^4.18.2",
"puppeteer": "^21.5.2"
"playwright": "^1.60.0",
"playwright-extra": "^4.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2"
}
}
}
simple-crawler/smart-cloner.js
+122 -12
View File
@@ -1,7 +1,8 @@
const fs = require('fs');
const path = require('path');
const puppeteer = require('puppeteer');
const { chromium } = require('playwright');
const { URL } = require('url');
const aiService = require('./ai-recreation');
class SmartCloner {
constructor() {
@@ -84,14 +85,15 @@ class SmartCloner {
const siteDir = path.join(this.baseOutputDir, `${siteName}_${timestamp}`);
this.ensureDirectoryExists(siteDir);
browser = await puppeteer.launch({
headless: 'new',
browser = await chromium.launch({
headless: true,
args: ['--no-sandbox', '--disable-setuid-sandbox', '--window-size=1920,1080']
});
const pagesToVisit = [url];
const visitedPages = new Set();
const pageMap = new Map(); // URL -> LocalFilename
const apiMocks = []; // Array of { url, method, status, headers, body }
// Pre-seed the map so links can be rewritten to files we haven't visited yet
pageMap.set(url.replace(/\/$/, ''), 'index.html');
@@ -108,16 +110,36 @@ class SmartCloner {
console.log(`📄 [${pageCount}/${maxPages}] Clonando: ${currentUrl}`);
const page = await browser.newPage();
await page.setViewport({ width: 1920, height: 1080 });
const context = await browser.newContext({ viewport: { width: 1920, height: 1080 }});
const page = await context.newPage();
// Asset interception
// Asset and API interception
page.on('response', async (response) => {
const rUrl = response.url();
if (rUrl.startsWith('data:') || !response.ok() || response.request().method() !== 'GET') return;
const req = response.request();
if (rUrl.startsWith('data:') || !response.ok()) return;
const resourceType = req.resourceType();
if (resourceType === 'fetch' || resourceType === 'xhr') {
try {
const bodyStr = await response.text();
apiMocks.push({
url: rUrl,
method: req.method(),
status: response.status(),
headers: response.headers(),
body: bodyStr
});
} catch (e) { }
return;
}
if (req.method() !== 'GET') return;
try {
const contentType = response.headers()['content-type'];
const buffer = await response.buffer();
const headers = response.headers();
const contentType = headers['content-type'];
const buffer = await response.body();
const localPath = this.generateLocalPath(rUrl, contentType, siteDir);
this.ensureDirectoryExists(path.dirname(localPath));
fs.writeFileSync(localPath, buffer);
@@ -126,10 +148,10 @@ class SmartCloner {
});
try {
await page.goto(currentUrl, { waitUntil: 'networkidle2', timeout: 60000 });
await page.goto(currentUrl, { waitUntil: 'networkidle', timeout: 60000 });
} catch (e) {
console.log(`⚠️ Falha ao carregar ${currentUrl}: ${e.message}`);
await page.close();
await context.close();
continue;
}
@@ -212,16 +234,104 @@ class SmartCloner {
})();
`;
document.body.appendChild(script);
// 4. Inject Service Worker Registration
const swScript = document.createElement('script');
swScript.textContent = `
if ('serviceWorker' in navigator) {
window.addEventListener('load', function() {
navigator.serviceWorker.register('/mock-sw.js').then(function(registration) {
console.log('Mock ServiceWorker registration successful with scope: ', registration.scope);
}, function(err) {
console.log('Mock ServiceWorker registration failed: ', err);
});
});
}
`;
document.body.appendChild(swScript);
}, resMapArr, pagMapArr);
// 5. AI Recreation of forms and dynamic content
const dynamicElements = await page.evaluate(() => {
return Array.from(document.querySelectorAll('form, [data-dynamic], .dynamic-content')).map(f => ({
html: f.outerHTML,
tagName: f.tagName
}));
});
for (const el of dynamicElements) {
const newHtml = await aiService.recreateComponent(el.html, el.tagName);
if (newHtml) {
await page.evaluate(({newHtml, oldHtml}) => {
const nodes = Array.from(document.querySelectorAll('form, [data-dynamic], .dynamic-content'));
const target = nodes.find(n => n.outerHTML === oldHtml);
if (target) {
target.outerHTML = newHtml;
}
}, { newHtml, oldHtml: el.html });
}
}
// 6. Expand Shadow DOM to Declarative Shadow DOM
await page.evaluate(() => {
function expandShadowRoots(node) {
if (node.shadowRoot) {
const template = document.createElement('template');
template.setAttribute('shadowrootmode', 'open');
template.innerHTML = node.shadowRoot.innerHTML;
node.insertBefore(template, node.firstChild);
// Recursively process inside shadowRoot
Array.from(node.shadowRoot.querySelectorAll('*')).forEach(expandShadowRoots);
}
Array.from(node.children).forEach(expandShadowRoots);
}
expandShadowRoots(document.body);
});
const html = await page.content();
const localFile = pageMap.get(cleanCurrentUrl) || 'index.html';
fs.writeFileSync(path.join(siteDir, localFile), html);
await page.close();
await context.close();
}
const info = { url, timestamp: new Date().toISOString(), pages: pageCount, directory: siteDir };
fs.writeFileSync(path.join(siteDir, 'clone-info.json'), JSON.stringify(info, null, 2));
// Save Mock DB
fs.writeFileSync(path.join(siteDir, 'mock-db.json'), JSON.stringify(apiMocks, null, 2));
// Create Service Worker
const swCode = `
const MOCK_DB_URL = '/mock-db.json';
let mockDb = null;
self.addEventListener('install', (event) => {
self.skipWaiting();
});
self.addEventListener('activate', (event) => {
event.waitUntil(clients.claim());
event.waitUntil(
fetch(MOCK_DB_URL).then(res => res.json()).then(data => { mockDb = data; })
);
});
self.addEventListener('fetch', (event) => {
if (event.request.method === 'GET' && !event.request.url.includes('/mock-db.json') && mockDb) {
const mock = mockDb.find(m => m.url === event.request.url || event.request.url.includes(m.url));
if (mock) {
event.respondWith(
new Response(mock.body, {
status: mock.status,
headers: { 'Content-Type': mock.headers['content-type'] || 'application/json' }
})
);
return;
}
}
});
`;
fs.writeFileSync(path.join(siteDir, 'mock-sw.js'), swCode);
return { success: true, path: siteDir, pagesCloned: pageCount };
} catch (e) {