import Bull from 'bull';
import Redis from 'ioredis';
import { IntelligentCrawler } from '../core/IntelligentCrawler';
import { CrawlJob, CrawlerConfig } from '../types/crawler';
import { logger } from '@cloneweb/shared';

const redis = new Redis(process.env.REDIS_URL || 'redis://localhost:6379');

export const crawlerQueue = new Bull('crawler', {
  redis: {
    port: 6379,
    host: process.env.REDIS_HOST || 'localhost',
  },
  defaultJobOptions: {
    removeOnComplete: 10,
    removeOnFail: 5,
    attempts: 3,
    backoff: {
      type: 'exponential',
      delay: 2000,
    },
  },
});

// Job processor
crawlerQueue.process('crawl-website', async (job) => {
  const { projectId, startUrl, config }: { projectId: string; startUrl: string; config: CrawlerConfig } = job.data;
  
  logger.info(`Starting crawl job for project ${projectId}: ${startUrl}`);
  
  const crawler = new IntelligentCrawler();
  
  try {
    // Update job progress
    await job.progress(10);
    
    // Initialize crawler
    await crawler.initialize();
    await job.progress(20);
    
    // Start crawling with progress updates
    const result = await crawler.crawlWebsite(startUrl, {
      ...config,
      // Add progress callback if needed
    });
    
    await job.progress(90);
    
    // Store result in database
    // TODO: Implement database storage
    
    await job.progress(100);
    
    logger.info(`Completed crawl job for project ${projectId}: ${result.totalPages} pages crawled`);
    
    return {
      projectId,
      siteMap: result,
      status: 'completed'
    };
    
  } catch (error) {
    logger.error(`Crawl job failed for project ${projectId}:`, error);
    throw error;
  } finally {
    await crawler.close();
  }
});

// Job event handlers
crawlerQueue.on('completed', (job, result) => {
  logger.info(`Job ${job.id} completed successfully`);
});

crawlerQueue.on('failed', (job, err) => {
  logger.error(`Job ${job.id} failed:`, err);
});

crawlerQueue.on('stalled', (job) => {
  logger.warn(`Job ${job.id} stalled`);
});

export const addCrawlJob = async (projectId: string, startUrl: string, config: CrawlerConfig): Promise<Bull.Job> => {
  return crawlerQueue.add('crawl-website', {
    projectId,
    startUrl,
    config
  }, {
    priority: 1,
    delay: 0,
  });
};