export interface CrawlerConfig {
  maxDepth: number;
  maxPages: number;
  respectRobots: boolean;
  rateLimit: number;
  userAgent: string;
  timeout: number;
  viewport: {
    width: number;
    height: number;
  };
  waitForSelector?: string;
  waitForTimeout?: number;
  captureScreenshot?: boolean;
}

export interface PageInfo {
  url: string;
  title: string;
  depth: number;
  links: string[];
  assets: AssetInfo[];
  metadata: PageMetadata;
  content: string;
  statusCode: number;
  loadTime: number;
  screenshot?: string;
}

export interface AssetInfo {
  url: string;
  type: 'image' | 'css' | 'js' | 'font' | 'media' | 'other';
  size?: number;
  hash?: string;
  dependencies: string[];
}

export interface PageMetadata {
  description?: string;
  keywords?: string[];
  author?: string;
  canonical?: string;
  ogTitle?: string;
  ogDescription?: string;
  ogImage?: string;
  twitterCard?: string;
  robots?: string;
  viewport?: string;
}

export interface SiteMap {
  rootUrl: string;
  pages: PageInfo[];
  totalPages: number;
  crawlDepth: number;
  crawlTime: number;
  structure: SiteStructure;
  metadata: SiteMetadata;
}

export interface SiteStructure {
  hierarchy: Map<string, string[]>;
  orphanPages: string[];
  externalLinks: string[];
}

export interface SiteMetadata {
  domain: string;
  robotsTxt?: string;
  sitemapXml?: string;
  favicon?: string;
  language?: string;
  charset?: string;
}

export interface CrawlJob {
  id: string;
  projectId: string;
  startUrl: string;
  config: CrawlerConfig;
  status: 'pending' | 'running' | 'completed' | 'failed';
  progress: number;
  createdAt: Date;
  updatedAt: Date;
  result?: SiteMap;
  error?: string;
}