PRIMEIRO ENVIO

test-crawler.js

@@ -0,0 +1,111 @@
+const express = require('express');
+const puppeteer = require('puppeteer');
+const cheerio = require('cheerio');
+
+const app = express();
+app.use(express.json());
+
+// Simplified crawler for testing
+class SimpleCrawler {
+  async crawlWebsite(startUrl, config = {}) {
+    const browser = await puppeteer.launch({
+      headless: 'new',
+      args: ['--no-sandbox', '--disable-setuid-sandbox']
+    });
+    
+    try {
+      const page = await browser.newPage();
+      await page.setViewport({ width: 1920, height: 1080 });
+      
+      console.log(`🕷️ Crawling: ${startUrl}`);
+      
+      const response = await page.goto(startUrl, {
+        waitUntil: 'networkidle2',
+        timeout: 30000
+      });
+      
+      // Wait for dynamic content
+      await page.waitForTimeout(2000);
+      
+      const content = await page.content();
+      const $ = cheerio.load(content);
+      
+      // Extract basic info
+      const title = $('title').text().trim();
+      const links = [];
+      
+      $('a[href]').each((_, element) => {
+        const href = $(element).attr('href');
+        if (href && href.startsWith('http')) {
+          links.push(href);
+        }
+      });
+      
+      const images = [];
+      $('img[src]').each((_, element) => {
+        const src = $(element).attr('src');
+        if (src) {
+          images.push(src);
+        }
+      });
+      
+      const result = {
+        url: startUrl,
+        title,
+        statusCode: response.status(),
+        links: [...new Set(links)].slice(0, 10), // First 10 unique links
+        images: [...new Set(images)].slice(0, 5), // First 5 unique images
+        contentLength: content.length,
+        timestamp: new Date().toISOString()
+      };
+      
+      console.log(`✅ Crawled successfully: ${title}`);
+      console.log(`📊 Found ${result.links.length} links and ${result.images.length} images`);
+      
+      return result;
+      
+    } finally {
+      await browser.close();
+    }
+  }
+}
+
+// API endpoint
+app.post('/crawl', async (req, res) => {
+  try {
+    const { url } = req.body;
+    
+    if (!url) {
+      return res.status(400).json({ error: 'URL is required' });
+    }
+    
+    console.log(`🚀 Starting crawl for: ${url}`);
+    
+    const crawler = new SimpleCrawler();
+    const result = await crawler.crawlWebsite(url);
+    
+    res.json({
+      success: true,
+      data: result
+    });
+    
+  } catch (error) {
+    console.error('❌ Crawl failed:', error.message);
+    res.status(500).json({
+      success: false,
+      error: error.message
+    });
+  }
+});
+
+// Health check
+app.get('/health', (req, res) => {
+  res.json({ status: 'ok', service: 'simple-crawler' });
+});
+
+// Start server
+const PORT = 3001;
+app.listen(PORT, () => {
+  console.log(`🌐 Simple Crawler running on http://localhost:${PORT}`);
+  console.log(`📋 Test with: curl -X POST http://localhost:${PORT}/crawl -H "Content-Type: application/json" -d '{"url":"https://example.com"}'`);
+});