PRIMEIRO ENVIO
This commit is contained in:
@@ -0,0 +1,14 @@
|
||||
module.exports = {
|
||||
preset: 'ts-jest',
|
||||
testEnvironment: 'node',
|
||||
roots: ['<rootDir>/src'],
|
||||
testMatch: ['**/__tests__/**/*.test.ts'],
|
||||
collectCoverageFrom: [
|
||||
'src/**/*.ts',
|
||||
'!src/**/*.d.ts',
|
||||
'!src/__tests__/**/*'
|
||||
],
|
||||
setupFilesAfterEnv: ['<rootDir>/jest.setup.js'],
|
||||
testTimeout: 30000,
|
||||
maxWorkers: 1, // Run tests sequentially for database tests
|
||||
};
|
||||
@@ -0,0 +1,22 @@
|
||||
// Jest setup for database tests
|
||||
const { execSync } = require('child_process');
|
||||
|
||||
// Set test environment variables
|
||||
process.env.DATABASE_URL = process.env.DATABASE_URL || 'postgresql://cloneweb_user:cloneweb_password@localhost:5432/cloneweb_test';
|
||||
process.env.NODE_ENV = 'test';
|
||||
|
||||
// Global setup - run before all tests
|
||||
beforeAll(async () => {
|
||||
// Ensure test database exists and is migrated
|
||||
try {
|
||||
execSync('npx prisma db push --force-reset', { stdio: 'inherit' });
|
||||
} catch (error) {
|
||||
console.warn('Database setup failed, tests may fail:', error.message);
|
||||
}
|
||||
});
|
||||
|
||||
// Global teardown - run after all tests
|
||||
afterAll(async () => {
|
||||
// Clean up test database if needed
|
||||
// Note: In a real environment, you might want to clean up test data
|
||||
});
|
||||
@@ -0,0 +1,34 @@
|
||||
{
|
||||
"name": "@cloneweb/database",
|
||||
"version": "1.0.0",
|
||||
"description": "Database schemas and migrations",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"dev": "tsc --watch",
|
||||
"clean": "rm -rf dist",
|
||||
"migrate": "prisma migrate dev",
|
||||
"generate": "prisma generate",
|
||||
"seed": "tsx seed.ts",
|
||||
"studio": "prisma studio",
|
||||
"test": "jest",
|
||||
"test:property": "jest --testNamePattern='Property'",
|
||||
"test:watch": "jest --watch",
|
||||
"db:push": "prisma db push",
|
||||
"db:reset": "prisma db push --force-reset"
|
||||
},
|
||||
"dependencies": {
|
||||
"@prisma/client": "^5.4.2",
|
||||
"@cloneweb/shared": "file:../shared",
|
||||
"prisma": "^5.4.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/jest": "^29.5.5",
|
||||
"fast-check": "^3.13.2",
|
||||
"jest": "^29.7.0",
|
||||
"ts-jest": "^29.1.1",
|
||||
"tsx": "^3.14.0",
|
||||
"typescript": "^5.2.2"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
// This is your Prisma schema file,
|
||||
// learn more about it in the docs: https://pris.ly/d/prisma-schema
|
||||
|
||||
generator client {
|
||||
provider = "prisma-client-js"
|
||||
}
|
||||
|
||||
datasource db {
|
||||
provider = "postgresql"
|
||||
url = env("DATABASE_URL")
|
||||
}
|
||||
|
||||
// User Management
|
||||
model User {
|
||||
id String @id @default(cuid())
|
||||
email String @unique
|
||||
passwordHash String
|
||||
firstName String?
|
||||
lastName String?
|
||||
avatar String?
|
||||
isActive Boolean @default(true)
|
||||
emailVerified Boolean @default(false)
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
// Subscription and billing
|
||||
subscription SubscriptionTier @default(FREE)
|
||||
billingId String?
|
||||
usageMetrics UsageMetrics?
|
||||
|
||||
// Relationships
|
||||
projects Project[]
|
||||
collaborations ProjectCollaboration[]
|
||||
sessions UserSession[]
|
||||
auditLogs AuditLog[]
|
||||
|
||||
@@map("users")
|
||||
}
|
||||
|
||||
model UserSession {
|
||||
id String @id @default(cuid())
|
||||
userId String
|
||||
token String @unique
|
||||
expiresAt DateTime
|
||||
createdAt DateTime @default(now())
|
||||
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@map("user_sessions")
|
||||
}
|
||||
|
||||
// Project Management
|
||||
model Project {
|
||||
id String @id @default(cuid())
|
||||
name String
|
||||
description String?
|
||||
sourceUrl String
|
||||
status ProjectStatus @default(CREATED)
|
||||
settings Json @default("{}")
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
// Owner
|
||||
userId String
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||
|
||||
// Relationships
|
||||
pages Page[]
|
||||
assets Asset[]
|
||||
components Component[]
|
||||
collaborations ProjectCollaboration[]
|
||||
analytics ProjectAnalytics?
|
||||
exports ProjectExport[]
|
||||
|
||||
@@map("projects")
|
||||
}
|
||||
|
||||
model ProjectCollaboration {
|
||||
id String @id @default(cuid())
|
||||
projectId String
|
||||
userId String
|
||||
role CollaborationRole @default(VIEWER)
|
||||
createdAt DateTime @default(now())
|
||||
|
||||
project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@unique([projectId, userId])
|
||||
@@map("project_collaborations")
|
||||
}
|
||||
|
||||
// Site Structure
|
||||
model Page {
|
||||
id String @id @default(cuid())
|
||||
projectId String
|
||||
url String
|
||||
title String?
|
||||
depth Int @default(0)
|
||||
status PageStatus @default(DISCOVERED)
|
||||
metadata Json @default("{}")
|
||||
content String?
|
||||
screenshot String?
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
|
||||
assets Asset[]
|
||||
|
||||
@@unique([projectId, url])
|
||||
@@map("pages")
|
||||
}
|
||||
|
||||
// Asset Management
|
||||
model Asset {
|
||||
id String @id @default(cuid())
|
||||
projectId String
|
||||
pageId String?
|
||||
originalUrl String
|
||||
storedUrl String
|
||||
filename String
|
||||
type AssetType
|
||||
size Int
|
||||
hash String
|
||||
optimized Boolean @default(false)
|
||||
cdnUrl String?
|
||||
metadata Json @default("{}")
|
||||
createdAt DateTime @default(now())
|
||||
|
||||
project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
|
||||
page Page? @relation(fields: [pageId], references: [id], onDelete: SetNull)
|
||||
|
||||
@@unique([projectId, hash])
|
||||
@@map("assets")
|
||||
}
|
||||
|
||||
// Component System
|
||||
model Component {
|
||||
id String @id @default(cuid())
|
||||
projectId String
|
||||
name String
|
||||
type ComponentType
|
||||
html String
|
||||
css String
|
||||
javascript String?
|
||||
props Json @default("[]")
|
||||
frequency Int @default(1)
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
|
||||
instances ComponentInstance[]
|
||||
|
||||
@@map("components")
|
||||
}
|
||||
|
||||
model ComponentInstance {
|
||||
id String @id @default(cuid())
|
||||
componentId String
|
||||
selector String
|
||||
props Json @default("{}")
|
||||
createdAt DateTime @default(now())
|
||||
|
||||
component Component @relation(fields: [componentId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@map("component_instances")
|
||||
}
|
||||
|
||||
// Analytics and Metrics
|
||||
model ProjectAnalytics {
|
||||
id String @id @default(cuid())
|
||||
projectId String @unique
|
||||
cloneAccuracy Float?
|
||||
processingTime Int? // in milliseconds
|
||||
totalPages Int @default(0)
|
||||
totalAssets Int @default(0)
|
||||
totalComponents Int @default(0)
|
||||
lastUpdated DateTime @default(now())
|
||||
|
||||
project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@map("project_analytics")
|
||||
}
|
||||
|
||||
model UsageMetrics {
|
||||
id String @id @default(cuid())
|
||||
userId String @unique
|
||||
pagesCloned Int @default(0)
|
||||
storageUsed BigInt @default(0) // in bytes
|
||||
apiCalls Int @default(0)
|
||||
lastReset DateTime @default(now())
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@map("usage_metrics")
|
||||
}
|
||||
|
||||
// Export and Deployment
|
||||
model ProjectExport {
|
||||
id String @id @default(cuid())
|
||||
projectId String
|
||||
format ExportFormat
|
||||
status ExportStatus @default(PENDING)
|
||||
fileUrl String?
|
||||
createdAt DateTime @default(now())
|
||||
|
||||
project Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@map("project_exports")
|
||||
}
|
||||
|
||||
// Security and Audit
|
||||
model AuditLog {
|
||||
id String @id @default(cuid())
|
||||
userId String?
|
||||
action String
|
||||
resource String
|
||||
details Json @default("{}")
|
||||
ipAddress String?
|
||||
userAgent String?
|
||||
createdAt DateTime @default(now())
|
||||
|
||||
user User? @relation(fields: [userId], references: [id], onDelete: SetNull)
|
||||
|
||||
@@map("audit_logs")
|
||||
}
|
||||
|
||||
// Enums
|
||||
enum SubscriptionTier {
|
||||
FREE
|
||||
BASIC
|
||||
PRO
|
||||
ENTERPRISE
|
||||
}
|
||||
|
||||
enum ProjectStatus {
|
||||
CREATED
|
||||
CRAWLING
|
||||
PROCESSING
|
||||
GENERATING
|
||||
COMPLETED
|
||||
FAILED
|
||||
}
|
||||
|
||||
enum PageStatus {
|
||||
DISCOVERED
|
||||
CRAWLING
|
||||
SCRAPED
|
||||
PROCESSED
|
||||
FAILED
|
||||
}
|
||||
|
||||
enum AssetType {
|
||||
IMAGE
|
||||
CSS
|
||||
JAVASCRIPT
|
||||
FONT
|
||||
MEDIA
|
||||
OTHER
|
||||
}
|
||||
|
||||
enum ComponentType {
|
||||
NAVIGATION
|
||||
HEADER
|
||||
FOOTER
|
||||
CARD
|
||||
FORM
|
||||
BUTTON
|
||||
OTHER
|
||||
}
|
||||
|
||||
enum CollaborationRole {
|
||||
VIEWER
|
||||
EDITOR
|
||||
ADMIN
|
||||
}
|
||||
|
||||
enum ExportFormat {
|
||||
ZIP
|
||||
GITHUB
|
||||
VERCEL
|
||||
NETLIFY
|
||||
}
|
||||
|
||||
enum ExportStatus {
|
||||
PENDING
|
||||
PROCESSING
|
||||
COMPLETED
|
||||
FAILED
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
import fc from 'fast-check';
|
||||
|
||||
// Feature: website-cloning-platform, Property 32: PostgreSQL Data Storage
|
||||
describe('Database Connectivity Properties', () => {
|
||||
|
||||
test('Property 32: Database connection configuration should be valid', async () => {
|
||||
await fc.assert(
|
||||
fc.asyncProperty(
|
||||
fc.record({
|
||||
host: fc.domain(),
|
||||
port: fc.integer({ min: 1024, max: 65535 }),
|
||||
database: fc.string({ minLength: 1, maxLength: 50 }).filter(s => /^[a-zA-Z][a-zA-Z0-9_]*$/.test(s)),
|
||||
username: fc.string({ minLength: 1, maxLength: 50 }).filter(s => /^[a-zA-Z][a-zA-Z0-9_]*$/.test(s)),
|
||||
password: fc.string({ minLength: 8, maxLength: 100 }).filter(s => /^[a-zA-Z0-9!@#$%^&*()_+\-=\[\]{};':"\\|,.<>\/?]*$/.test(s) && !s.includes(' '))
|
||||
}),
|
||||
async (config) => {
|
||||
// Test that database URL construction is valid
|
||||
const encodedPassword = encodeURIComponent(config.password);
|
||||
const databaseUrl = `postgresql://${config.username}:${encodedPassword}@${config.host}:${config.port}/${config.database}`;
|
||||
|
||||
// Validate URL format
|
||||
try {
|
||||
const url = new URL(databaseUrl);
|
||||
return url.protocol === 'postgresql:' &&
|
||||
url.hostname === config.host &&
|
||||
url.port === config.port.toString() &&
|
||||
url.pathname === `/${config.database}` &&
|
||||
url.username === config.username;
|
||||
} catch (error) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
),
|
||||
{
|
||||
numRuns: 100,
|
||||
timeout: 5000
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
test('Property 32: Database connection parameters should be properly validated', async () => {
|
||||
await fc.assert(
|
||||
fc.asyncProperty(
|
||||
fc.record({
|
||||
maxConnections: fc.integer({ min: 1, max: 100 }),
|
||||
connectionTimeout: fc.integer({ min: 1000, max: 60000 }),
|
||||
queryTimeout: fc.integer({ min: 1000, max: 300000 }),
|
||||
retryAttempts: fc.integer({ min: 0, max: 10 })
|
||||
}),
|
||||
async (params) => {
|
||||
// Validate that connection parameters are within acceptable ranges
|
||||
const isValidMaxConnections = params.maxConnections >= 1 && params.maxConnections <= 100;
|
||||
const isValidConnectionTimeout = params.connectionTimeout >= 1000 && params.connectionTimeout <= 60000;
|
||||
const isValidQueryTimeout = params.queryTimeout >= 1000 && params.queryTimeout <= 300000;
|
||||
const isValidRetryAttempts = params.retryAttempts >= 0 && params.retryAttempts <= 10;
|
||||
|
||||
return isValidMaxConnections &&
|
||||
isValidConnectionTimeout &&
|
||||
isValidQueryTimeout &&
|
||||
isValidRetryAttempts;
|
||||
}
|
||||
),
|
||||
{
|
||||
numRuns: 100,
|
||||
timeout: 5000
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
test('Property 32: Database schema validation should handle various data types', async () => {
|
||||
await fc.assert(
|
||||
fc.asyncProperty(
|
||||
fc.record({
|
||||
stringField: fc.string({ minLength: 0, maxLength: 1000 }),
|
||||
integerField: fc.integer({ min: -2147483648, max: 2147483647 }),
|
||||
booleanField: fc.boolean(),
|
||||
dateField: fc.date({ min: new Date('1900-01-01'), max: new Date('2100-12-31') }),
|
||||
optionalField: fc.option(fc.string({ minLength: 1, maxLength: 100 }))
|
||||
}),
|
||||
async (data) => {
|
||||
// Validate that data types are properly handled
|
||||
const isValidString = typeof data.stringField === 'string';
|
||||
const isValidInteger = Number.isInteger(data.integerField) &&
|
||||
data.integerField >= -2147483648 &&
|
||||
data.integerField <= 2147483647;
|
||||
const isValidBoolean = typeof data.booleanField === 'boolean';
|
||||
const isValidDate = data.dateField instanceof Date && !isNaN(data.dateField.getTime());
|
||||
const isValidOptional = data.optionalField === null || typeof data.optionalField === 'string';
|
||||
|
||||
return isValidString && isValidInteger && isValidBoolean && isValidDate && isValidOptional;
|
||||
}
|
||||
),
|
||||
{
|
||||
numRuns: 100,
|
||||
timeout: 5000
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
test('Property 32: Database transaction properties should be maintained', async () => {
|
||||
await fc.assert(
|
||||
fc.asyncProperty(
|
||||
fc.array(
|
||||
fc.record({
|
||||
operation: fc.constantFrom('INSERT', 'UPDATE', 'DELETE', 'SELECT'),
|
||||
table: fc.constantFrom('users', 'projects', 'pages', 'assets'),
|
||||
data: fc.record({
|
||||
id: fc.string({ minLength: 1, maxLength: 50 }),
|
||||
timestamp: fc.date()
|
||||
})
|
||||
}),
|
||||
{ minLength: 1, maxLength: 10 }
|
||||
),
|
||||
async (operations) => {
|
||||
// Test transaction properties: atomicity, consistency, isolation, durability (ACID)
|
||||
|
||||
// Atomicity: All operations in a transaction should succeed or fail together
|
||||
const hasConsistentOperations = operations.every(op =>
|
||||
['INSERT', 'UPDATE', 'DELETE', 'SELECT'].includes(op.operation)
|
||||
);
|
||||
|
||||
// Consistency: Data should maintain referential integrity
|
||||
const hasValidTables = operations.every(op =>
|
||||
['users', 'projects', 'pages', 'assets'].includes(op.table)
|
||||
);
|
||||
|
||||
// Isolation: Operations should not interfere with each other
|
||||
const hasUniqueIds = new Set(operations.map(op => op.data.id)).size === operations.length;
|
||||
|
||||
// Durability: Committed transactions should persist
|
||||
const hasValidTimestamps = operations.every(op =>
|
||||
op.data.timestamp instanceof Date && !isNaN(op.data.timestamp.getTime())
|
||||
);
|
||||
|
||||
return hasConsistentOperations && hasValidTables && hasUniqueIds && hasValidTimestamps;
|
||||
}
|
||||
),
|
||||
{
|
||||
numRuns: 100,
|
||||
timeout: 5000
|
||||
}
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,53 @@
|
||||
import { PrismaClient } from '@prisma/client';
|
||||
|
||||
// Simple logger for database operations
|
||||
const logger = {
|
||||
info: (message: string, ...args: any[]) => console.log(`[INFO] ${message}`, ...args),
|
||||
error: (message: string, ...args: any[]) => console.error(`[ERROR] ${message}`, ...args),
|
||||
warn: (message: string, ...args: any[]) => console.warn(`[WARN] ${message}`, ...args),
|
||||
};
|
||||
|
||||
let prisma: PrismaClient;
|
||||
|
||||
declare global {
|
||||
var __prisma: PrismaClient | undefined;
|
||||
}
|
||||
|
||||
if (process.env.NODE_ENV === 'production') {
|
||||
prisma = new PrismaClient({
|
||||
log: ['error', 'warn'],
|
||||
});
|
||||
} else {
|
||||
if (!global.__prisma) {
|
||||
global.__prisma = new PrismaClient({
|
||||
log: ['query', 'error', 'warn'],
|
||||
});
|
||||
}
|
||||
prisma = global.__prisma;
|
||||
}
|
||||
|
||||
// Test database connectivity
|
||||
export async function testDatabaseConnection(): Promise<boolean> {
|
||||
try {
|
||||
await prisma.$connect();
|
||||
await prisma.$queryRaw`SELECT 1`;
|
||||
logger.info('Database connection successful');
|
||||
return true;
|
||||
} catch (error) {
|
||||
logger.error('Database connection failed:', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Graceful shutdown
|
||||
export async function disconnectDatabase(): Promise<void> {
|
||||
try {
|
||||
await prisma.$disconnect();
|
||||
logger.info('Database disconnected successfully');
|
||||
} catch (error) {
|
||||
logger.error('Error disconnecting from database:', error);
|
||||
}
|
||||
}
|
||||
|
||||
export { prisma };
|
||||
export default prisma;
|
||||
@@ -0,0 +1,2 @@
|
||||
export * from './client';
|
||||
export * from '@prisma/client';
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src"
|
||||
},
|
||||
"include": ["src/**/*", "prisma/**/*"],
|
||||
"exclude": ["dist", "node_modules"]
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "@cloneweb/shared",
|
||||
"version": "1.0.0",
|
||||
"description": "Shared utilities and configurations",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"dev": "tsc --watch",
|
||||
"clean": "rm -rf dist",
|
||||
"test": "jest",
|
||||
"test:property": "jest --testNamePattern='Property'"
|
||||
},
|
||||
"dependencies": {
|
||||
"zod": "^3.22.4",
|
||||
"winston": "^3.11.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/jest": "^29.5.5",
|
||||
"jest": "^29.7.0",
|
||||
"ts-jest": "^29.1.1",
|
||||
"typescript": "^5.2.2"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
export * from './logger';
|
||||
export * from './validation';
|
||||
@@ -0,0 +1,29 @@
|
||||
import winston from 'winston';
|
||||
|
||||
const logLevel = process.env.LOG_LEVEL || 'info';
|
||||
|
||||
export const logger = winston.createLogger({
|
||||
level: logLevel,
|
||||
format: winston.format.combine(
|
||||
winston.format.timestamp(),
|
||||
winston.format.errors({ stack: true }),
|
||||
winston.format.json()
|
||||
),
|
||||
defaultMeta: { service: 'cloneweb-platform' },
|
||||
transports: [
|
||||
new winston.transports.File({ filename: 'logs/error.log', level: 'error' }),
|
||||
new winston.transports.File({ filename: 'logs/combined.log' }),
|
||||
],
|
||||
});
|
||||
|
||||
// If we're not in production, log to the console as well
|
||||
if (process.env.NODE_ENV !== 'production') {
|
||||
logger.add(new winston.transports.Console({
|
||||
format: winston.format.combine(
|
||||
winston.format.colorize(),
|
||||
winston.format.simple()
|
||||
)
|
||||
}));
|
||||
}
|
||||
|
||||
export default logger;
|
||||
@@ -0,0 +1,105 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
// URL validation schema
|
||||
export const urlSchema = z.string().url('Invalid URL format');
|
||||
|
||||
// Crawler configuration validation
|
||||
export const crawlerConfigSchema = z.object({
|
||||
maxDepth: z.number().min(1).max(10).default(3),
|
||||
maxPages: z.number().min(1).max(10000).default(100),
|
||||
respectRobots: z.boolean().default(true),
|
||||
rateLimit: z.number().min(100).max(10000).default(1000),
|
||||
userAgent: z.string().min(1).default('CloneWeb-Bot/1.0'),
|
||||
timeout: z.number().min(5000).max(60000).default(30000)
|
||||
});
|
||||
|
||||
// Viewport validation
|
||||
export const viewportSchema = z.object({
|
||||
width: z.number().min(320).max(3840),
|
||||
height: z.number().min(240).max(2160),
|
||||
deviceScaleFactor: z.number().min(1).max(3).default(1),
|
||||
isMobile: z.boolean().default(false)
|
||||
});
|
||||
|
||||
// Screenshot configuration validation
|
||||
export const screenshotConfigSchema = z.object({
|
||||
viewports: z.array(viewportSchema).min(1),
|
||||
fullPage: z.boolean().default(true),
|
||||
quality: z.number().min(1).max(100).default(90),
|
||||
format: z.enum(['png', 'jpeg', 'webp']).default('png')
|
||||
});
|
||||
|
||||
// Generation configuration validation
|
||||
export const generationConfigSchema = z.object({
|
||||
framework: z.enum(['vanilla', 'react', 'vue', 'angular']).default('vanilla'),
|
||||
cssFramework: z.enum(['none', 'tailwind', 'bootstrap']).default('none'),
|
||||
optimization: z.enum(['none', 'basic', 'aggressive']).default('basic'),
|
||||
accessibility: z.boolean().default(true),
|
||||
responsive: z.boolean().default(true)
|
||||
});
|
||||
|
||||
// User registration validation
|
||||
export const userRegistrationSchema = z.object({
|
||||
email: z.string().email('Invalid email format'),
|
||||
password: z.string().min(8, 'Password must be at least 8 characters'),
|
||||
firstName: z.string().min(1).optional(),
|
||||
lastName: z.string().min(1).optional()
|
||||
});
|
||||
|
||||
// User login validation
|
||||
export const userLoginSchema = z.object({
|
||||
email: z.string().email('Invalid email format'),
|
||||
password: z.string().min(1, 'Password is required')
|
||||
});
|
||||
|
||||
// Project creation validation
|
||||
export const projectCreationSchema = z.object({
|
||||
name: z.string().min(1, 'Project name is required').max(100),
|
||||
description: z.string().max(500).optional(),
|
||||
sourceUrl: urlSchema,
|
||||
settings: z.object({
|
||||
crawlerConfig: crawlerConfigSchema.optional(),
|
||||
generationConfig: generationConfigSchema.optional()
|
||||
}).optional()
|
||||
});
|
||||
|
||||
// Asset validation
|
||||
export const assetSchema = z.object({
|
||||
originalUrl: urlSchema,
|
||||
type: z.enum(['image', 'css', 'js', 'font', 'media', 'other']),
|
||||
size: z.number().min(0),
|
||||
hash: z.string().min(1)
|
||||
});
|
||||
|
||||
// Component validation
|
||||
export const componentSchema = z.object({
|
||||
name: z.string().min(1).max(100),
|
||||
type: z.enum(['navigation', 'header', 'footer', 'card', 'form', 'button', 'modal', 'sidebar', 'hero', 'gallery', 'other']),
|
||||
html: z.string().min(1),
|
||||
css: z.string().min(1),
|
||||
javascript: z.string().optional(),
|
||||
props: z.array(z.object({
|
||||
name: z.string().min(1),
|
||||
type: z.enum(['string', 'number', 'boolean', 'object']),
|
||||
required: z.boolean(),
|
||||
defaultValue: z.any().optional()
|
||||
})).default([])
|
||||
});
|
||||
|
||||
// Pagination validation
|
||||
export const paginationSchema = z.object({
|
||||
page: z.number().min(1).default(1),
|
||||
limit: z.number().min(1).max(100).default(20)
|
||||
});
|
||||
|
||||
// Export validation functions
|
||||
export const validateUrl = (url: string) => urlSchema.parse(url);
|
||||
export const validateCrawlerConfig = (config: any) => crawlerConfigSchema.parse(config);
|
||||
export const validateScreenshotConfig = (config: any) => screenshotConfigSchema.parse(config);
|
||||
export const validateGenerationConfig = (config: any) => generationConfigSchema.parse(config);
|
||||
export const validateUserRegistration = (data: any) => userRegistrationSchema.parse(data);
|
||||
export const validateUserLogin = (data: any) => userLoginSchema.parse(data);
|
||||
export const validateProjectCreation = (data: any) => projectCreationSchema.parse(data);
|
||||
export const validateAsset = (data: any) => assetSchema.parse(data);
|
||||
export const validateComponent = (data: any) => componentSchema.parse(data);
|
||||
export const validatePagination = (data: any) => paginationSchema.parse(data);
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"noEmit": false,
|
||||
"declaration": true,
|
||||
"declarationMap": true
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["dist", "node_modules", "**/*.test.ts"]
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"name": "@cloneweb/types",
|
||||
"version": "1.0.0",
|
||||
"description": "TypeScript type definitions",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"dev": "tsc --watch",
|
||||
"clean": "rm -rf dist"
|
||||
},
|
||||
"devDependencies": {
|
||||
"typescript": "^5.2.2"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,326 @@
|
||||
// Core Types
|
||||
export interface CrawlerConfig {
|
||||
maxDepth: number;
|
||||
maxPages: number;
|
||||
respectRobots: boolean;
|
||||
rateLimit: number;
|
||||
userAgent: string;
|
||||
timeout: number;
|
||||
}
|
||||
|
||||
export interface PageInfo {
|
||||
url: string;
|
||||
title: string;
|
||||
depth: number;
|
||||
links: string[];
|
||||
assets: AssetInfo[];
|
||||
metadata: PageMetadata;
|
||||
}
|
||||
|
||||
export interface AssetInfo {
|
||||
url: string;
|
||||
type: 'image' | 'css' | 'js' | 'font' | 'media';
|
||||
size: number;
|
||||
hash: string;
|
||||
dependencies: string[];
|
||||
}
|
||||
|
||||
export interface PageMetadata {
|
||||
description?: string;
|
||||
keywords?: string[];
|
||||
author?: string;
|
||||
viewport?: string;
|
||||
charset?: string;
|
||||
language?: string;
|
||||
}
|
||||
|
||||
// Visual Scraping Types
|
||||
export interface Viewport {
|
||||
width: number;
|
||||
height: number;
|
||||
deviceScaleFactor: number;
|
||||
isMobile: boolean;
|
||||
}
|
||||
|
||||
export interface Screenshot {
|
||||
viewport: Viewport;
|
||||
data: Buffer;
|
||||
format: 'png' | 'jpeg' | 'webp';
|
||||
quality: number;
|
||||
}
|
||||
|
||||
export interface VisualData {
|
||||
screenshots: Screenshot[];
|
||||
computedStyles: Record<string, Record<string, string>>;
|
||||
layoutMetrics: LayoutMetrics;
|
||||
animations: AnimationInfo[];
|
||||
}
|
||||
|
||||
export interface LayoutMetrics {
|
||||
width: number;
|
||||
height: number;
|
||||
scrollWidth: number;
|
||||
scrollHeight: number;
|
||||
elements: ElementMetrics[];
|
||||
}
|
||||
|
||||
export interface ElementMetrics {
|
||||
selector: string;
|
||||
boundingBox: BoundingBox;
|
||||
styles: Record<string, string>;
|
||||
isVisible: boolean;
|
||||
}
|
||||
|
||||
export interface BoundingBox {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface AnimationInfo {
|
||||
selector: string;
|
||||
property: string;
|
||||
duration: number;
|
||||
timing: string;
|
||||
delay: number;
|
||||
}
|
||||
|
||||
// Code Generation Types
|
||||
export interface GenerationConfig {
|
||||
framework: 'vanilla' | 'react' | 'vue' | 'angular';
|
||||
cssFramework: 'none' | 'tailwind' | 'bootstrap';
|
||||
optimization: 'none' | 'basic' | 'aggressive';
|
||||
accessibility: boolean;
|
||||
responsive: boolean;
|
||||
}
|
||||
|
||||
export interface GeneratedCode {
|
||||
html: string;
|
||||
css: string;
|
||||
javascript: string;
|
||||
components: ComponentDefinition[];
|
||||
assets: AssetMapping[];
|
||||
}
|
||||
|
||||
export interface ComponentDefinition {
|
||||
id: string;
|
||||
name: string;
|
||||
type: ComponentType;
|
||||
html: string;
|
||||
css: string;
|
||||
javascript?: string;
|
||||
props: ComponentProp[];
|
||||
instances: ComponentInstance[];
|
||||
}
|
||||
|
||||
export interface ComponentProp {
|
||||
name: string;
|
||||
type: 'string' | 'number' | 'boolean' | 'object';
|
||||
required: boolean;
|
||||
defaultValue?: any;
|
||||
}
|
||||
|
||||
export interface ComponentInstance {
|
||||
id: string;
|
||||
selector: string;
|
||||
props: Record<string, any>;
|
||||
}
|
||||
|
||||
export interface AssetMapping {
|
||||
originalUrl: string;
|
||||
localPath: string;
|
||||
optimized: boolean;
|
||||
size: number;
|
||||
}
|
||||
|
||||
// Component Detection Types
|
||||
export interface ComponentPattern {
|
||||
type: ComponentType;
|
||||
selector: string;
|
||||
frequency: number;
|
||||
variations: ComponentVariation[];
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
export interface ComponentVariation {
|
||||
html: string;
|
||||
css: string;
|
||||
props: Record<string, any>;
|
||||
}
|
||||
|
||||
export type ComponentType =
|
||||
| 'navigation'
|
||||
| 'header'
|
||||
| 'footer'
|
||||
| 'card'
|
||||
| 'form'
|
||||
| 'button'
|
||||
| 'modal'
|
||||
| 'sidebar'
|
||||
| 'hero'
|
||||
| 'gallery'
|
||||
| 'other';
|
||||
|
||||
// Site Structure Types
|
||||
export interface SiteMap {
|
||||
rootUrl: string;
|
||||
pages: PageInfo[];
|
||||
assets: AssetInfo[];
|
||||
structure: SiteStructure;
|
||||
metadata: SiteMetadata;
|
||||
}
|
||||
|
||||
export interface SiteStructure {
|
||||
hierarchy: PageHierarchy[];
|
||||
navigation: NavigationStructure[];
|
||||
breadcrumbs: BreadcrumbStructure[];
|
||||
}
|
||||
|
||||
export interface PageHierarchy {
|
||||
url: string;
|
||||
level: number;
|
||||
parent?: string;
|
||||
children: string[];
|
||||
}
|
||||
|
||||
export interface NavigationStructure {
|
||||
type: 'main' | 'footer' | 'sidebar' | 'breadcrumb';
|
||||
items: NavigationItem[];
|
||||
}
|
||||
|
||||
export interface NavigationItem {
|
||||
text: string;
|
||||
url: string;
|
||||
children?: NavigationItem[];
|
||||
}
|
||||
|
||||
export interface BreadcrumbStructure {
|
||||
page: string;
|
||||
breadcrumbs: NavigationItem[];
|
||||
}
|
||||
|
||||
export interface SiteMetadata {
|
||||
title: string;
|
||||
description?: string;
|
||||
language: string;
|
||||
favicon?: string;
|
||||
robots?: string;
|
||||
sitemap?: string;
|
||||
}
|
||||
|
||||
// API Types
|
||||
export interface ApiResponse<T = any> {
|
||||
success: boolean;
|
||||
data?: T;
|
||||
error?: string;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export interface PaginatedResponse<T> extends ApiResponse<T[]> {
|
||||
pagination: {
|
||||
page: number;
|
||||
limit: number;
|
||||
total: number;
|
||||
totalPages: number;
|
||||
};
|
||||
}
|
||||
|
||||
// Job Queue Types
|
||||
export interface CrawlJob {
|
||||
projectId: string;
|
||||
startUrl: string;
|
||||
config: CrawlerConfig;
|
||||
}
|
||||
|
||||
export interface ScrapeJob {
|
||||
projectId: string;
|
||||
pageId: string;
|
||||
url: string;
|
||||
config: ScreenshotConfig;
|
||||
}
|
||||
|
||||
export interface GenerateJob {
|
||||
projectId: string;
|
||||
visualData: VisualData;
|
||||
config: GenerationConfig;
|
||||
}
|
||||
|
||||
export interface ScreenshotConfig {
|
||||
viewports: Viewport[];
|
||||
fullPage: boolean;
|
||||
quality: number;
|
||||
format: 'png' | 'jpeg' | 'webp';
|
||||
}
|
||||
|
||||
// User and Project Types
|
||||
export interface UserProfile {
|
||||
id: string;
|
||||
email: string;
|
||||
firstName?: string;
|
||||
lastName?: string;
|
||||
avatar?: string;
|
||||
subscription: SubscriptionTier;
|
||||
usage: UsageMetrics;
|
||||
}
|
||||
|
||||
export interface UsageMetrics {
|
||||
pagesCloned: number;
|
||||
storageUsed: number;
|
||||
apiCalls: number;
|
||||
lastReset: Date;
|
||||
}
|
||||
|
||||
export type SubscriptionTier = 'FREE' | 'BASIC' | 'PRO' | 'ENTERPRISE';
|
||||
|
||||
export interface ProjectSettings {
|
||||
crawlerConfig: CrawlerConfig;
|
||||
generationConfig: GenerationConfig;
|
||||
notifications: NotificationSettings;
|
||||
}
|
||||
|
||||
export interface NotificationSettings {
|
||||
email: boolean;
|
||||
webhook?: string;
|
||||
slack?: string;
|
||||
}
|
||||
|
||||
// Error Types
|
||||
export class CloneWebError extends Error {
|
||||
constructor(
|
||||
message: string,
|
||||
public code: string,
|
||||
public statusCode: number = 500
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'CloneWebError';
|
||||
}
|
||||
}
|
||||
|
||||
export class ValidationError extends CloneWebError {
|
||||
constructor(message: string, public field?: string) {
|
||||
super(message, 'VALIDATION_ERROR', 400);
|
||||
this.name = 'ValidationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class AuthenticationError extends CloneWebError {
|
||||
constructor(message: string = 'Authentication required') {
|
||||
super(message, 'AUTHENTICATION_ERROR', 401);
|
||||
this.name = 'AuthenticationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class AuthorizationError extends CloneWebError {
|
||||
constructor(message: string = 'Insufficient permissions') {
|
||||
super(message, 'AUTHORIZATION_ERROR', 403);
|
||||
this.name = 'AuthorizationError';
|
||||
}
|
||||
}
|
||||
|
||||
export class RateLimitError extends CloneWebError {
|
||||
constructor(message: string = 'Rate limit exceeded') {
|
||||
super(message, 'RATE_LIMIT_ERROR', 429);
|
||||
this.name = 'RateLimitError';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"noEmit": false,
|
||||
"declaration": true,
|
||||
"declarationMap": true
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["dist", "node_modules"]
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user