import { chromium, Browser, Page, BrowserContext } from 'playwright';
import { URL } from 'url';
import * as cheerio from 'cheerio';
import robotsParser from 'robots-parser';
import { logger } from '@cloneweb/shared';
import { CrawlerConfig, PageInfo, SiteMap, AssetInfo, PageMetadata, SiteStructure } from '../types/crawler';

export class IntelligentCrawler {
  private browser: Browser | null = null;
  private visitedUrls = new Set<string>();
  private urlQueue: Array<{ url: string; depth: number }> = [];
  private robotsCache = new Map<string, any>();
  private lastRequestTime = 0;

  async initialize(): Promise<void> {
    this.browser = await chromium.launch({
      headless: true,
      args: [
        '--no-sandbox',
        '--disable-setuid-sandbox',
        '--disable-dev-shm-usage',
        '--disable-accelerated-2d-canvas',
        '--no-first-run',
        '--no-zygote',
        '--single-process',
        '--disable-gpu'
      ]
    });
    logger.info('Playwright browser initialized');
  }

  async crawlWebsite(startUrl: string, config: CrawlerConfig): Promise<SiteMap> {
    const startTime = Date.now();
    
    if (!this.browser) {
      await this.initialize();
    }

    // Initialize crawl state
    this.visitedUrls.clear();
    this.urlQueue = [{ url: startUrl, depth: 0 }];
    
    const pages: PageInfo[] = [];
    const baseUrl = new URL(startUrl);
    
    // Check robots.txt if required
    if (config.respectRobots) {
      await this.loadRobotsTxt(baseUrl.origin, config.userAgent);
    }

    // BFS crawling algorithm
    while (this.urlQueue.length > 0 && pages.length < config.maxPages) {
      const { url, depth } = this.urlQueue.shift()!;
      
      if (this.visitedUrls.has(url) || depth > config.maxDepth) {
        continue;
      }

      // Rate limiting
      await this.respectRateLimit(config.rateLimit);

      try {
        const pageInfo = await this.extractPageInfo(url, depth, config);
        if (pageInfo) {
          pages.push(pageInfo);
          this.visitedUrls.add(url);
          
          // Add discovered links to queue
          this.addLinksToQueue(pageInfo.links, depth + 1, baseUrl.origin);
          
          logger.info(`Crawled page ${pages.length}/${config.maxPages}: ${url}`);
        }
      } catch (error) {
        logger.error(`Failed to crawl ${url}:`, error);
      }
    }

    const crawlTime = Date.now() - startTime;
    
    return {
      rootUrl: startUrl,
      pages,
      totalPages: pages.length,
      crawlDepth: Math.max(...pages.map(p => p.depth)),
      crawlTime,
      structure: this.buildSiteStructure(pages, startUrl),
      metadata: await this.extractSiteMetadata(baseUrl.origin)
    };
  }

  async extractPageInfo(url: string, depth: number, config: CrawlerConfig): Promise<PageInfo | null> {
    const context = await this.browser!.newContext({
      viewport: config.viewport,
      userAgent: config.userAgent
    });
    const page = await context.newPage();
    
    try {
      const startTime = Date.now();
      
      // Navigate to page
      const response = await page.goto(url, {
        waitUntil: 'networkidle',
        timeout: config.timeout
      });

      if (!response) {
        throw new Error('No response received');
      }

      const loadTime = Date.now() - startTime;
      
      // Wait for additional content if specified
      if (config.waitForSelector) {
        await page.waitForSelector(config.waitForSelector, { timeout: 5000 }).catch(() => {});
      }
      
      if (config.waitForTimeout) {
        await page.waitForTimeout(config.waitForTimeout);
      }

      // Execute JavaScript and wait for dynamic content
      await this.executeJavaScriptAndWaitForContent(page);

      // Extract page content after JavaScript execution
      const content = await page.content();
      const $ = cheerio.load(content);
      
      // Extract metadata
      const metadata = this.extractMetadata($);
      
      // Extract links
      const links = this.extractLinks($, url);
      
      // Extract assets
      const assets = this.extractAssets($, url);
      
      // Get page title
      const title = $('title').text().trim() || new URL(url).pathname;

      // Capture screenshot if needed
      let screenshot: string | undefined;
      if (config.captureScreenshot) {
        const screenshotBuffer = await page.screenshot({
          fullPage: true,
          type: 'png'
        });
        screenshot = screenshotBuffer.toString('base64');
      }

      return {
        url,
        title,
        depth,
        links,
        assets,
        metadata,
        content,
        statusCode: response.status(),
        loadTime,
        screenshot
      };
      
    } finally {
      await context.close();
    }
  }

  private async executeJavaScriptAndWaitForContent(page: Page): Promise<void> {
    try {
      // Wait for common dynamic content indicators
      await Promise.race([
        // Wait for React/Vue apps to mount
        page.waitForFunction(() => {
          return document.querySelector('[data-reactroot], #app, .vue-app, [ng-app]') !== null;
        }, { timeout: 3000 }).catch(() => {}),
        
        // Wait for AJAX requests to complete
        page.waitForFunction(() => {
          return (window as any).jQuery ? (window as any).jQuery.active === 0 : true;
        }, { timeout: 3000 }).catch(() => {}),
        
        // Wait for loading indicators to disappear
        page.waitForFunction(() => {
          const loaders = document.querySelectorAll('.loading, .spinner, .loader, [class*="loading"]');
          return Array.from(loaders).every(loader => 
            getComputedStyle(loader).display === 'none' || 
            getComputedStyle(loader).visibility === 'hidden'
          );
        }, { timeout: 3000 }).catch(() => {}),
        
        // Fallback timeout
        page.waitForTimeout(2000)
      ]);

      // Scroll to trigger lazy loading
      await page.evaluate(() => {
        return new Promise<void>((resolve) => {
          let totalHeight = 0;
          const distance = 100;
          const timer = setInterval(() => {
            const scrollHeight = document.body.scrollHeight;
            window.scrollBy(0, distance);
            totalHeight += distance;

            if (totalHeight >= scrollHeight) {
              clearInterval(timer);
              window.scrollTo(0, 0); // Scroll back to top
              setTimeout(resolve, 500); // Wait a bit more for content to load
            }
          }, 100);
        });
      });

    } catch (error) {
      logger.warn(`JavaScript execution failed for page: ${page.url()}`, error);
    }
  }

  private async loadRobotsTxt(origin: string, userAgent: string): Promise<void> {
    if (this.robotsCache.has(origin)) {
      return;
    }

    try {
      const robotsUrl = `${origin}/robots.txt`;
      const context = await this.browser!.newContext();
      const page = await context.newPage();
      
      const response = await page.goto(robotsUrl, { timeout: 10000 });
      
      if (response && response.ok()) {
        const robotsTxt = await page.content();
        const robots = robotsParser(robotsUrl, robotsTxt);
        this.robotsCache.set(origin, robots);
        logger.info(`Loaded robots.txt for ${origin}`);
      }
      
      await context.close();
    } catch (error) {
      logger.warn(`Failed to load robots.txt for ${origin}:`, error);
      this.robotsCache.set(origin, null);
    }
  }

  private canCrawlUrl(url: string, userAgent: string): boolean {
    const urlObj = new URL(url);
    const robots = this.robotsCache.get(urlObj.origin);
    
    if (!robots) {
      return true; // No robots.txt or failed to load
    }
    
    return robots.isAllowed(url, userAgent);
  }

  private async respectRateLimit(rateLimit: number): Promise<void> {
    const now = Date.now();
    const timeSinceLastRequest = now - this.lastRequestTime;
    const minInterval = 1000 / rateLimit; // Convert requests per second to milliseconds
    
    if (timeSinceLastRequest < minInterval) {
      const delay = minInterval - timeSinceLastRequest;
      await new Promise(resolve => setTimeout(resolve, delay));
    }
    
    this.lastRequestTime = Date.now();
  }

  private extractMetadata($: cheerio.CheerioAPI): PageMetadata {
    return {
      description: $('meta[name="description"]').attr('content'),
      keywords: $('meta[name="keywords"]').attr('content')?.split(',').map(k => k.trim()),
      author: $('meta[name="author"]').attr('content'),
      canonical: $('link[rel="canonical"]').attr('href'),
      ogTitle: $('meta[property="og:title"]').attr('content'),
      ogDescription: $('meta[property="og:description"]').attr('content'),
      ogImage: $('meta[property="og:image"]').attr('content'),
      twitterCard: $('meta[name="twitter:card"]').attr('content'),
      robots: $('meta[name="robots"]').attr('content'),
      viewport: $('meta[name="viewport"]').attr('content')
    };
  }

  private extractLinks($: cheerio.CheerioAPI, baseUrl: string): string[] {
    const links: string[] = [];
    const base = new URL(baseUrl);
    
    $('a[href]').each((_, element) => {
      const href = $(element).attr('href');
      if (href) {
        try {
          const absoluteUrl = new URL(href, baseUrl).href;
          const urlObj = new URL(absoluteUrl);
          
          // Only include same-origin links
          if (urlObj.origin === base.origin) {
            links.push(absoluteUrl);
          }
        } catch (error) {
          // Invalid URL, skip
        }
      }
    });
    
    return [...new Set(links)]; // Remove duplicates
  }

  private extractAssets($: cheerio.CheerioAPI, baseUrl: string): AssetInfo[] {
    const assets: AssetInfo[] = [];
    
    // Images
    $('img[src]').each((_, element) => {
      const src = $(element).attr('src');
      if (src) {
        assets.push({
          url: new URL(src, baseUrl).href,
          type: 'image',
          dependencies: []
        });
      }
    });
    
    // CSS files
    $('link[rel="stylesheet"][href]').each((_, element) => {
      const href = $(element).attr('href');
      if (href) {
        assets.push({
          url: new URL(href, baseUrl).href,
          type: 'css',
          dependencies: []
        });
      }
    });
    
    // JavaScript files
    $('script[src]').each((_, element) => {
      const src = $(element).attr('src');
      if (src) {
        assets.push({
          url: new URL(src, baseUrl).href,
          type: 'js',
          dependencies: []
        });
      }
    });
    
    // Fonts
    $('link[rel="preload"][as="font"], link[rel="font"]').each((_, element) => {
      const href = $(element).attr('href');
      if (href) {
        assets.push({
          url: new URL(href, baseUrl).href,
          type: 'font',
          dependencies: []
        });
      }
    });
    
    return assets;
  }

  private addLinksToQueue(links: string[], depth: number, origin: string): void {
    for (const link of links) {
      if (!this.visitedUrls.has(link) && !this.urlQueue.some(item => item.url === link)) {
        // Check robots.txt if available
        if (this.robotsCache.has(origin)) {
          const robots = this.robotsCache.get(origin);
          if (robots && !robots.isAllowed(link, 'CloneWeb-Bot')) {
            continue;
          }
        }
        
        this.urlQueue.push({ url: link, depth });
      }
    }
  }

  private buildSiteStructure(pages: PageInfo[], rootUrl: string): SiteStructure {
    const hierarchy = new Map<string, string[]>();
    const allUrls = new Set(pages.map(p => p.url));
    const externalLinks: string[] = [];
    
    for (const page of pages) {
      const internalLinks = page.links.filter(link => allUrls.has(link));
      const external = page.links.filter(link => !allUrls.has(link));
      
      hierarchy.set(page.url, internalLinks);
      externalLinks.push(...external);
    }
    
    // Find orphan pages (pages with no incoming links except root)
    const linkedPages = new Set<string>();
    for (const links of hierarchy.values()) {
      links.forEach(link => linkedPages.add(link));
    }
    
    const orphanPages = pages
      .filter(p => p.url !== rootUrl && !linkedPages.has(p.url))
      .map(p => p.url);
    
    return {
      hierarchy,
      orphanPages,
      externalLinks: [...new Set(externalLinks)]
    };
  }

  generateSitemap(siteMap: SiteMap): string {
    const { pages, rootUrl } = siteMap;
    
    // Generate XML sitemap
    let xml = '<?xml version="1.0" encoding="UTF-8"?>\n';
    xml += '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n';
    
    for (const page of pages) {
      xml += '  <url>\n';
      xml += `    <loc>${page.url}</loc>\n`;
      xml += `    <lastmod>${new Date().toISOString().split('T')[0]}</lastmod>\n`;
      
      // Set priority based on depth (root = 1.0, deeper pages = lower priority)
      const priority = Math.max(0.1, 1.0 - (page.depth * 0.2));
      xml += `    <priority>${priority.toFixed(1)}</priority>\n`;
      
      // Set change frequency based on depth
      const changefreq = page.depth === 0 ? 'daily' : 
                        page.depth === 1 ? 'weekly' : 'monthly';
      xml += `    <changefreq>${changefreq}</changefreq>\n`;
      
      xml += '  </url>\n';
    }
    
    xml += '</urlset>';
    return xml;
  }

  generateSiteHierarchy(siteMap: SiteMap): any {
    const { pages, structure } = siteMap;
    
    // Build hierarchical representation
    const hierarchy: any = {
      url: siteMap.rootUrl,
      title: pages.find(p => p.url === siteMap.rootUrl)?.title || 'Root',
      depth: 0,
      children: []
    };
    
    const buildTree = (parentUrl: string, currentDepth: number): any[] => {
      const children = structure.hierarchy.get(parentUrl) || [];
      
      return children.map(childUrl => {
        const childPage = pages.find(p => p.url === childUrl);
        if (!childPage) return null;
        
        return {
          url: childUrl,
          title: childPage.title,
          depth: childPage.depth,
          children: buildTree(childUrl, childPage.depth)
        };
      }).filter(Boolean);
    };
    
    hierarchy.children = buildTree(siteMap.rootUrl, 0);
    
    return {
      hierarchy,
      orphanPages: structure.orphanPages.map(url => {
        const page = pages.find(p => p.url === url);
        return {
          url,
          title: page?.title || 'Unknown',
          depth: page?.depth || 0
        };
      }),
      externalLinks: structure.externalLinks,
      statistics: {
        totalPages: pages.length,
        maxDepth: Math.max(...pages.map(p => p.depth)),
        avgLinksPerPage: pages.reduce((sum, p) => sum + p.links.length, 0) / pages.length,
        totalAssets: pages.reduce((sum, p) => sum + p.assets.length, 0)
      }
    };
  }

  private async extractSiteMetadata(origin: string): Promise<any> {
    // This would extract site-level metadata like robots.txt, sitemap.xml, etc.
    return {
      domain: new URL(origin).hostname,
      robotsTxt: this.robotsCache.get(origin)?.toString(),
      language: 'en', // Could be detected from pages
      charset: 'utf-8'
    };
  }

  async close(): Promise<void> {
    if (this.browser) {
      await this.browser.close();
      this.browser = null;
    }
  }
}