PRIMEIRO ENVIO

services/crawler/src/__tests__/crawler.property.test.ts

@@ -0,0 +1,379 @@
+import * as fc from 'fast-check';
+import { IntelligentCrawler } from '../core/IntelligentCrawler';
+import { CrawlerConfig } from '../types/crawler';
+
+// Mock the shared logger
+jest.mock('@cloneweb/shared', () => ({
+  logger: {
+    info: jest.fn(),
+    error: jest.fn(),
+    warn: jest.fn(),
+    debug: jest.fn(),
+    level: 'error'
+  }
+}));
+
+// Feature: website-cloning-platform, Property 1: BFS Page Discovery
+describe('Crawler Property Tests', () => {
+  let crawler: IntelligentCrawler;
+
+  beforeEach(() => {
+    crawler = new IntelligentCrawler();
+  });
+
+  afterEach(async () => {
+    await crawler.close();
+  });
+
+  // Property 1: BFS Page Discovery
+  test('Property 1: BFS Page Discovery - should discover all reachable pages in breadth-first order', async () => {
+    // Custom generator for website structure
+    const websiteStructureGenerator = () => fc.record({
+      rootUrl: fc.webUrl(),
+      pages: fc.array(fc.record({
+        url: fc.webUrl(),
+        depth: fc.integer({ min: 1, max: 3 }), // Start from depth 1 (children of root)
+        links: fc.array(fc.webUrl(), { maxLength: 3 })
+      }), { minLength: 1, maxLength: 8 })
+    });
+
+    const defaultConfig: CrawlerConfig = {
+      maxDepth: 3,
+      maxPages: 50,
+      respectRobots: false, // Disable for testing
+      rateLimit: 10,
+      userAgent: 'Test-Bot/1.0',
+      timeout: 5000,
+      viewport: { width: 1280, height: 720 }
+    };
+
+    await fc.assert(fc.asyncProperty(
+      websiteStructureGenerator(),
+      async (websiteStructure) => {
+        // Mock the crawler for testing
+        const mockCrawler = new MockIntelligentCrawler();
+        mockCrawler.setMockWebsite(websiteStructure);
+        
+        const result = await mockCrawler.crawlWebsite(websiteStructure.rootUrl, defaultConfig);
+        
+        // Verify BFS order: pages at depth n should be discovered before pages at depth n+1
+        const pagesByDepth = new Map<number, string[]>();
+        result.pages.forEach((page: any) => {
+          if (!pagesByDepth.has(page.depth)) {
+            pagesByDepth.set(page.depth, []);
+          }
+          pagesByDepth.get(page.depth)!.push(page.url);
+        });
+        
+        // Check that depths are reasonable (should start from 0 for root)
+        const depths = Array.from(pagesByDepth.keys()).sort((a, b) => a - b);
+        
+        // Should have at least the root page at depth 0
+        expect(depths[0]).toBe(0);
+        
+        // Check that there are no unreasonable gaps in depths
+        // (allow some gaps since not all depths may be represented in small test cases)
+        for (let i = 1; i < depths.length; i++) {
+          const gap = depths[i] - depths[i - 1];
+          expect(gap).toBeLessThanOrEqual(3); // Allow reasonable gaps
+        }
+        
+        // Verify all discovered pages are within max depth
+        result.pages.forEach((page: any) => {
+          expect(page.depth).toBeLessThanOrEqual(defaultConfig.maxDepth);
+        });
+        
+        // Verify we found at least the root page
+        expect(result.pages.length).toBeGreaterThanOrEqual(1);
+        expect(result.pages.some((p: any) => p.url === websiteStructure.rootUrl)).toBe(true);
+        
+        return true;
+      }
+    ), { numRuns: 20, timeout: 30000 });
+  });
+
+  // Property 2: Rate Limiting Compliance
+  test('Property 2: Rate Limiting Compliance - should respect configured rate limits', async () => {
+    const rateLimitGenerator = () => fc.record({
+      rateLimit: fc.integer({ min: 1, max: 5 }), // requests per second
+      urls: fc.array(fc.webUrl(), { minLength: 3, maxLength: 6 }).map(urls => {
+        // Ensure unique URLs to avoid deduplication
+        return Array.from(new Set(urls)).slice(0, 4);
+      })
+    });
+
+    await fc.assert(fc.asyncProperty(
+      rateLimitGenerator(),
+      async ({ rateLimit, urls }) => {
+        // Skip if we don't have enough unique URLs
+        if (urls.length < 2) return true;
+        
+        const config: CrawlerConfig = {
+          maxDepth: 2,
+          maxPages: urls.length,
+          respectRobots: false,
+          rateLimit,
+          userAgent: 'Test-Bot/1.0',
+          timeout: 5000,
+          viewport: { width: 1280, height: 720 }
+        };
+
+        const mockCrawler = new MockIntelligentCrawler();
+        mockCrawler.setMockWebsite({
+          rootUrl: urls[0],
+          pages: urls.slice(1).map((url, index) => ({
+            url,
+            depth: 1,
+            links: []
+          }))
+        });
+
+        const startTime = Date.now();
+        await mockCrawler.crawlWebsite(urls[0], config);
+        const endTime = Date.now();
+        
+        const actualDuration = endTime - startTime;
+        // Calculate minimum expected duration for the number of requests after the first
+        const requestCount = Math.min(urls.length, config.maxPages);
+        const minExpectedDuration = (requestCount - 1) * (1000 / rateLimit);
+        
+        // Allow some tolerance for test execution overhead (50% tolerance)
+        expect(actualDuration).toBeGreaterThanOrEqual(minExpectedDuration * 0.5);
+        
+        return true;
+      }
+    ), { numRuns: 10, timeout: 60000 });
+  });
+
+  // Property 3: JavaScript Content Capture
+  test('Property 3: JavaScript Content Capture - should capture JavaScript-generated content', async () => {
+    const jsContentGenerator = () => fc.record({
+      url: fc.webUrl(),
+      hasJavaScript: fc.boolean(),
+      dynamicElements: fc.array(fc.string(), { minLength: 0, maxLength: 5 })
+    });
+
+    const defaultConfig: CrawlerConfig = {
+      maxDepth: 1,
+      maxPages: 1,
+      respectRobots: false,
+      rateLimit: 5,
+      userAgent: 'Test-Bot/1.0',
+      timeout: 10000,
+      viewport: { width: 1280, height: 720 },
+      waitForTimeout: 1000
+    };
+
+    await fc.assert(fc.asyncProperty(
+      jsContentGenerator(),
+      async ({ url, hasJavaScript, dynamicElements }) => {
+        const mockCrawler = new MockIntelligentCrawler();
+        
+        // Set up mock website with JavaScript content
+        mockCrawler.setMockWebsite({
+          rootUrl: url,
+          pages: [],
+          hasJavaScript,
+          dynamicElements
+        });
+        
+        const result = await mockCrawler.crawlWebsite(url, defaultConfig);
+        
+        // Verify that the page was captured
+        expect(result.pages.length).toBeGreaterThanOrEqual(1);
+        
+        const rootPage = result.pages.find((p: any) => p.url === url);
+        expect(rootPage).toBeDefined();
+        
+        // Verify content was captured
+        expect(rootPage.content).toBeDefined();
+        expect(rootPage.content.length).toBeGreaterThan(0);
+        
+        // If JavaScript is present, verify dynamic content handling
+        if (hasJavaScript && dynamicElements.length > 0) {
+          // Mock should simulate that dynamic content was captured
+          expect(rootPage.content).toContain('dynamic-content-loaded');
+        }
+        
+        return true;
+      }
+    ), { numRuns: 15, timeout: 30000 });
+  });
+
+  // Property 4: Complete Sitemap Generation
+  test('Property 4: Complete Sitemap Generation - should generate complete sitemap with hierarchical relationships', async () => {
+    const sitemapGenerator = () => fc.record({
+      rootUrl: fc.webUrl(),
+      pages: fc.array(fc.record({
+        url: fc.webUrl(),
+        depth: fc.integer({ min: 1, max: 3 }),
+        links: fc.array(fc.webUrl(), { maxLength: 3 })
+      }), { minLength: 1, maxLength: 8 })
+    });
+
+    const defaultConfig: CrawlerConfig = {
+      maxDepth: 3,
+      maxPages: 20,
+      respectRobots: false,
+      rateLimit: 10,
+      userAgent: 'Test-Bot/1.0',
+      timeout: 5000,
+      viewport: { width: 1280, height: 720 }
+    };
+
+    await fc.assert(fc.asyncProperty(
+      sitemapGenerator(),
+      async (websiteStructure) => {
+        const mockCrawler = new MockIntelligentCrawler();
+        mockCrawler.setMockWebsite(websiteStructure);
+        
+        const result = await mockCrawler.crawlWebsite(websiteStructure.rootUrl, defaultConfig);
+        
+        // Verify sitemap completeness
+        expect(result.pages.length).toBeGreaterThanOrEqual(1);
+        expect(result.structure).toBeDefined();
+        expect(result.metadata).toBeDefined();
+        
+        // Verify hierarchical relationships
+        expect(result.structure.hierarchy).toBeDefined();
+        expect(result.structure.orphanPages).toBeDefined();
+        expect(result.structure.externalLinks).toBeDefined();
+        
+        // Generate XML sitemap
+        const xmlSitemap = mockCrawler.generateSitemap(result);
+        expect(xmlSitemap).toContain('<?xml version="1.0" encoding="UTF-8"?>');
+        expect(xmlSitemap).toContain('<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">');
+        expect(xmlSitemap).toContain(websiteStructure.rootUrl);
+        
+        // Generate hierarchical sitemap
+        const hierarchicalSitemap = mockCrawler.generateSiteHierarchy(result);
+        expect(hierarchicalSitemap.hierarchy).toBeDefined();
+        expect(hierarchicalSitemap.hierarchy.url).toBe(websiteStructure.rootUrl);
+        expect(hierarchicalSitemap.statistics).toBeDefined();
+        expect(hierarchicalSitemap.statistics.totalPages).toBe(result.pages.length);
+        
+        // Verify all discovered pages are included in sitemap
+        result.pages.forEach((page: any) => {
+          expect(xmlSitemap).toContain(page.url);
+        });
+        
+        return true;
+      }
+    ), { numRuns: 15, timeout: 30000 });
+  });
+});
+
+// Mock implementation for testing
+class MockIntelligentCrawler extends IntelligentCrawler {
+  private mockWebsite: any;
+  private requestTimes: number[] = [];
+
+  setMockWebsite(website: any) {
+    this.mockWebsite = website;
+  }
+
+  async initialize(): Promise<void> {
+    // Mock initialization - no real browser needed
+  }
+
+  async crawlWebsite(startUrl: string, config: CrawlerConfig): Promise<any> {
+    const startTime = Date.now();
+    const pages: any[] = [];
+    const visitedUrls = new Set<string>();
+    const urlQueue: Array<{ url: string; depth: number }> = [{ url: startUrl, depth: 0 }];
+
+    // Add all mock pages to the queue
+    if (this.mockWebsite.pages) {
+      this.mockWebsite.pages.forEach((page: any) => {
+        if (!urlQueue.some(item => item.url === page.url)) {
+          urlQueue.push({ url: page.url, depth: page.depth });
+        }
+      });
+    }
+
+    while (urlQueue.length > 0 && pages.length < config.maxPages) {
+      const { url, depth } = urlQueue.shift()!;
+      
+      if (visitedUrls.has(url) || depth > config.maxDepth) {
+        continue;
+      }
+
+      // Apply rate limiting
+      await this.respectRateLimit(config.rateLimit);
+
+      const pageInfo = await this.extractPageInfo(url, depth, config);
+      if (pageInfo) {
+        pages.push(pageInfo);
+        visitedUrls.add(url);
+      }
+    }
+
+    const crawlTime = Date.now() - startTime;
+    
+    return {
+      rootUrl: startUrl,
+      pages,
+      totalPages: pages.length,
+      crawlDepth: Math.max(...pages.map((p: any) => p.depth)),
+      crawlTime,
+      structure: { hierarchy: new Map(), orphanPages: [], externalLinks: [] },
+      metadata: { domain: 'test.com' }
+    };
+  }
+
+  private async respectRateLimit(rateLimit: number): Promise<void> {
+    const now = Date.now();
+    this.requestTimes.push(now);
+    
+    if (this.requestTimes.length > 1) {
+      const timeSinceLastRequest = now - this.requestTimes[this.requestTimes.length - 2];
+      const minInterval = 1000 / rateLimit;
+      
+      if (timeSinceLastRequest < minInterval) {
+        const delay = minInterval - timeSinceLastRequest;
+        await new Promise(resolve => setTimeout(resolve, delay));
+        // Update the recorded time after the delay
+        this.requestTimes[this.requestTimes.length - 1] = Date.now();
+      }
+    }
+  }
+
+  async extractPageInfo(url: string, depth: number, config: CrawlerConfig): Promise<any> {
+    // Find the mock page
+    const mockPage = this.mockWebsite.pages?.find((p: any) => p.url === url) || {
+      url,
+      depth,
+      links: []
+    };
+
+    // Simulate JavaScript content capture
+    let content = `<html><head><title>Page ${url}</title></head><body>Content`;
+    
+    // Add dynamic content if JavaScript is enabled
+    if (this.mockWebsite.hasJavaScript && this.mockWebsite.dynamicElements?.length > 0) {
+      content += '<div class="dynamic-content-loaded">';
+      this.mockWebsite.dynamicElements.forEach((element: string) => {
+        content += `<span class="dynamic">${element}</span>`;
+      });
+      content += '</div>';
+    }
+    
+    content += '</body></html>';
+
+    return {
+      url,
+      title: `Page ${url}`,
+      depth,
+      links: mockPage.links || [],
+      assets: [],
+      metadata: {},
+      content,
+      statusCode: 200,
+      loadTime: 100
+    };
+  }
+
+  async close(): Promise<void> {
+    // Mock close - nothing to clean up
+  }
+}