#!/usr/bin/env tsx /** * Project Scanner Script * * Scans ~/projects/ directory for Git repos and monorepos, * then adds them as Tenants (clients) in the Sankofa system. * * Features: * - Detects Git repositories and monorepos * - Extracts project metadata (Git remote, branch, package manager, workspaces) * - Creates Tenants via GraphQL API with proper authentication * - Duplicate detection and validation * - Progress reporting and structured logging * * Usage: * tsx scripts/scan-projects.ts [options] * * Options: * --dry-run Show what would be created without actually creating * --skip-existing Skip projects that already exist as tenants * --projects-dir=PATH Directory to scan (default: ~/projects/) * --api-url=URL GraphQL API URL (default: http://localhost:4000/graphql) * --email=EMAIL Email for authentication (or set GRAPHQL_EMAIL env var) * --password=PASSWORD Password for authentication (or set GRAPHQL_PASSWORD env var) * --auth-token=TOKEN Direct authentication token (or set GRAPHQL_AUTH_TOKEN env var) * --output=json Output results as JSON * --verbose Enable verbose logging * --batch-size=N Number of projects to process in parallel (default: 1) */ import { readdir, stat, readFile } from 'fs/promises' import { join, basename } from 'path' import { existsSync } from 'fs' import { execSync } from 'child_process' import { program } from 'commander' import cliProgress from 'cli-progress' import * as dotenv from 'dotenv' import { z } from 'zod' // Load environment variables dotenv.config() // Validation schemas const projectNameSchema = z.string().min(1).max(255).regex(/^[a-zA-Z0-9_-]+$/, { message: 'Project name must contain only alphanumeric characters, hyphens, and underscores' }) const gitUrlSchema = z.string().url().or(z.literal('')) interface ProjectInfo { name: string path: string type: 'repo' | 'monorepo' | 'other' gitRemote?: string gitBranch?: string packageManager?: 'npm' | 'pnpm' | 'yarn' | 'unknown' workspaces?: string[] metadata: Record } interface GraphQLResponse { data?: T errors?: Array<{ message: string; extensions?: { code?: string } }> } interface AuthPayload { token: string user: { id: string email: string name: string role: string } } interface Tenant { id: string name: string domain?: string | null } interface ScanOptions { projectsDir: string dryRun: boolean skipExisting: boolean apiUrl: string email?: string password?: string authToken?: string outputFormat: 'text' | 'json' verbose: boolean batchSize: number } interface ScanResult { created: Array<{ name: string; id: string }> skipped: Array<{ name: string; reason: string }> errors: Array<{ name: string; error: string }> } class ProjectScanner { private options: ScanOptions private authToken?: string private existingTenants: Map = new Map() constructor(options: ScanOptions) { this.options = options } /** * Authenticate with the GraphQL API */ async authenticate(): Promise { if (this.options.authToken) { this.authToken = this.options.authToken if (this.options.verbose) { console.log('Using provided authentication token') } return } if (!this.options.email || !this.options.password) { throw new Error( 'Authentication required. Provide --email and --password, or --auth-token, ' + 'or set GRAPHQL_EMAIL/GRAPHQL_PASSWORD or GRAPHQL_AUTH_TOKEN environment variables' ) } const mutation = ` mutation Login($email: String!, $password: String!) { login(email: $email, password: $password) { token user { id email name role } } } ` try { const response = await this.executeGraphQL<{ login: AuthPayload }>( mutation, { email: this.options.email, password: this.options.password }, 'login' ) if (!response) { throw new Error('Login failed: No response received') } this.authToken = response.login.token if (response.login.user.role !== 'ADMIN') { throw new Error( `Admin role required. Current role: ${response.login.user.role}. ` + 'Only ADMIN users can create tenants.' ) } if (this.options.verbose) { console.log(`Authenticated as ${response.login.user.email} (${response.login.user.role})`) } } catch (error) { throw new Error(`Authentication failed: ${error instanceof Error ? error.message : String(error)}`) } } /** * Load existing tenants for duplicate detection */ async loadExistingTenants(): Promise { if (this.options.dryRun || !this.options.skipExisting) { return } const query = ` query ListTenants { tenants { id name domain } } ` try { const response = await this.executeGraphQL<{ tenants: Tenant[] }>( query, {}, 'tenants' ) if (response) { for (const tenant of response.tenants) { this.existingTenants.set(tenant.name.toLowerCase(), tenant) if (tenant.domain) { this.existingTenants.set(tenant.domain.toLowerCase(), tenant) } } if (this.options.verbose) { console.log(`Loaded ${this.existingTenants.size} existing tenants for duplicate detection`) } } } catch (error) { if (this.options.verbose) { console.warn(`Warning: Could not load existing tenants: ${error instanceof Error ? error.message : String(error)}`) } } } /** * Check if a directory is a Git repository */ private async isGitRepo(dirPath: string): Promise { return existsSync(join(dirPath, '.git')) } /** * Get Git remote URL if available */ private async getGitRemote(dirPath: string): Promise { try { const result = execSync('git remote get-url origin', { cwd: dirPath, encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'], }) const url = result.trim() // Validate URL gitUrlSchema.parse(url) return url } catch { return undefined } } /** * Get current Git branch */ private async getGitBranch(dirPath: string): Promise { try { const result = execSync('git rev-parse --abbrev-ref HEAD', { cwd: dirPath, encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'], }) return result.trim() } catch { return undefined } } /** * Extract domain from Git remote URL */ private extractDomainFromGitRemote(remote?: string): string | undefined { if (!remote) return undefined try { // Handle various Git URL formats // https://github.com/user/repo.git -> github.com // git@github.com:user/repo.git -> github.com // https://gitlab.com/user/repo.git -> gitlab.com // https://bitbucket.org/user/repo.git -> bitbucket.org const patterns = [ /^https?:\/\/([^\/]+)\//, /^git@([^:]+):/, /^ssh:\/\/git@([^\/]+)\//, ] for (const pattern of patterns) { const match = remote.match(pattern) if (match && match[1]) { const host = match[1].replace(/^www\./, '') // Generate a safe domain name return `${host.replace(/\./g, '-')}.dev` } } } catch { // Ignore errors } return undefined } /** * Detect package manager from lock files */ private detectPackageManager(dirPath: string): 'npm' | 'pnpm' | 'yarn' | 'unknown' { if (existsSync(join(dirPath, 'pnpm-lock.yaml'))) return 'pnpm' if (existsSync(join(dirPath, 'yarn.lock'))) return 'yarn' if (existsSync(join(dirPath, 'package-lock.json'))) return 'npm' return 'unknown' } /** * Check if a directory is a monorepo */ private async isMonorepo(dirPath: string): Promise { const packageJsonPath = join(dirPath, 'package.json') if (!existsSync(packageJsonPath)) return false try { const content = await readFile(packageJsonPath, 'utf-8') const pkg = JSON.parse(content) // Check for workspaces (npm/yarn) if (pkg.workspaces && Array.isArray(pkg.workspaces) && pkg.workspaces.length > 0) { return true } // Check for pnpm workspace if (existsSync(join(dirPath, 'pnpm-workspace.yaml'))) { return true } // Check for lerna if (existsSync(join(dirPath, 'lerna.json'))) { return true } // Check for nx if (existsSync(join(dirPath, 'nx.json'))) { return true } // Check for turborepo if (existsSync(join(dirPath, 'turbo.json'))) { return true } // Check for multiple package.json files in subdirectories const entries = await readdir(dirPath, { withFileTypes: true }) let packageJsonCount = 0 for (const entry of entries) { if (entry.isDirectory() && !entry.name.startsWith('.')) { if (existsSync(join(dirPath, entry.name, 'package.json'))) { packageJsonCount++ if (packageJsonCount >= 2) return true } } } } catch { // If we can't read/parse package.json, it's not a monorepo } return false } /** * Get workspace packages for a monorepo */ private async getWorkspaces(dirPath: string): Promise { const workspaces: string[] = [] const packageJsonPath = join(dirPath, 'package.json') if (existsSync(packageJsonPath)) { try { const content = await readFile(packageJsonPath, 'utf-8') const pkg = JSON.parse(content) if (pkg.workspaces && Array.isArray(pkg.workspaces)) { // Expand workspace patterns for (const pattern of pkg.workspaces) { const glob = pattern.replace('/*', '').replace('*', '') const workspaceDir = join(dirPath, glob) if (existsSync(workspaceDir)) { const entries = await readdir(workspaceDir, { withFileTypes: true }) for (const entry of entries) { if (entry.isDirectory()) { const workspacePath = join(workspaceDir, entry.name) if (existsSync(join(workspacePath, 'package.json'))) { workspaces.push(entry.name) } } } } } } } catch { // Ignore errors } } // Check pnpm-workspace.yaml const pnpmWorkspacePath = join(dirPath, 'pnpm-workspace.yaml') if (existsSync(pnpmWorkspacePath)) { try { const content = await readFile(pnpmWorkspacePath, 'utf-8') // Simple parsing for pnpm workspace const lines = content.split('\n') for (const line of lines) { const match = line.match(/^[\s-]+['"]?([^'"]+)['"]?/) if (match) { workspaces.push(match[1]) } } } catch { // Ignore errors } } return workspaces } /** * Validate and sanitize project name */ private validateProjectName(name: string): string { // Remove invalid characters and replace with hyphens let sanitized = name.replace(/[^a-zA-Z0-9_-]/g, '-') // Remove consecutive hyphens sanitized = sanitized.replace(/-+/g, '-') // Remove leading/trailing hyphens sanitized = sanitized.replace(/^-+|-+$/g, '') // Ensure it's not empty if (!sanitized) { sanitized = 'project' } // Truncate to max length if (sanitized.length > 255) { sanitized = sanitized.substring(0, 255) } // Validate with schema projectNameSchema.parse(sanitized) return sanitized } /** * Sanitize metadata to prevent injection and size issues */ private sanitizeMetadata(metadata: Record): Record { const sanitized: Record = {} const maxSize = 10 * 1024 // 10KB limit for (const [key, value] of Object.entries(metadata)) { // Only allow string, number, boolean, null, and arrays/objects of these types if ( typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean' || value === null ) { sanitized[key] = value } else if (Array.isArray(value)) { sanitized[key] = value.filter( (v) => typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean' || v === null ) } else if (typeof value === 'object') { // Recursively sanitize nested objects sanitized[key] = this.sanitizeMetadata(value as Record) } } // Check size const jsonString = JSON.stringify(sanitized) if (jsonString.length > maxSize) { // Truncate by removing less important fields delete sanitized.description delete sanitized.keywords const truncated = JSON.stringify(sanitized) if (truncated.length > maxSize) { throw new Error(`Metadata too large (${jsonString.length} bytes, max ${maxSize} bytes)`) } } return sanitized } /** * Scan a directory and collect project information */ private async scanDirectory(dirPath: string): Promise { const name = basename(dirPath) const stats = await stat(dirPath) if (!stats.isDirectory()) { return null } // Skip hidden directories and common non-project directories if (name.startsWith('.') || ['node_modules', 'dist', 'build', '.git'].includes(name)) { return null } const isGit = await this.isGitRepo(dirPath) const isMono = await this.isMonorepo(dirPath) // Only process Git repos or directories with package.json if (!isGit && !existsSync(join(dirPath, 'package.json'))) { return null } // Validate and sanitize project name const sanitizedName = this.validateProjectName(name) const projectInfo: ProjectInfo = { name: sanitizedName, path: dirPath, type: isMono ? 'monorepo' : isGit ? 'repo' : 'other', metadata: {}, } if (isGit) { projectInfo.gitRemote = await this.getGitRemote(dirPath) projectInfo.gitBranch = await this.getGitBranch(dirPath) } if (existsSync(join(dirPath, 'package.json'))) { projectInfo.packageManager = this.detectPackageManager(dirPath) if (isMono) { projectInfo.workspaces = await this.getWorkspaces(dirPath) } // Read package.json for additional metadata try { const content = await readFile(join(dirPath, 'package.json'), 'utf-8') const pkg = JSON.parse(content) projectInfo.metadata = { description: pkg.description, version: pkg.version, author: typeof pkg.author === 'string' ? pkg.author : pkg.author?.name, license: pkg.license, keywords: Array.isArray(pkg.keywords) ? pkg.keywords.slice(0, 10) : undefined, } } catch { // Ignore errors } } // Sanitize metadata projectInfo.metadata = this.sanitizeMetadata(projectInfo.metadata) return projectInfo } /** * Scan all projects in the projects directory */ async scanAllProjects(): Promise { const projects: ProjectInfo[] = [] if (!existsSync(this.options.projectsDir)) { throw new Error(`Projects directory does not exist: ${this.options.projectsDir}`) } if (this.options.verbose) { console.log(`Scanning projects in: ${this.options.projectsDir}`) } const entries = await readdir(this.options.projectsDir, { withFileTypes: true }) for (const entry of entries) { if (entry.isDirectory()) { const projectPath = join(this.options.projectsDir, entry.name) try { const projectInfo = await this.scanDirectory(projectPath) if (projectInfo) { projects.push(projectInfo) } } catch (error) { if (this.options.verbose) { console.warn(`Warning: Failed to scan ${entry.name}: ${error instanceof Error ? error.message : String(error)}`) } } } } return projects } /** * Check if tenant already exists */ private isDuplicate(project: ProjectInfo): Tenant | null { const nameKey = project.name.toLowerCase() if (this.existingTenants.has(nameKey)) { return this.existingTenants.get(nameKey)! } if (project.gitRemote) { const domain = this.extractDomainFromGitRemote(project.gitRemote) if (domain) { const domainKey = domain.toLowerCase() if (this.existingTenants.has(domainKey)) { return this.existingTenants.get(domainKey)! } } } return null } /** * Create a Tenant via GraphQL */ private async createTenant(project: ProjectInfo): Promise { const mutation = ` mutation CreateTenant($input: CreateTenantInput!) { createTenant(input: $input) { id name domain status tier } } ` const domain = this.extractDomainFromGitRemote(project.gitRemote) const variables = { input: { name: project.name, domain: domain || undefined, tier: 'STANDARD' as const, metadata: { type: project.type, path: project.path, gitRemote: project.gitRemote, gitBranch: project.gitBranch, packageManager: project.packageManager, workspaces: project.workspaces, ...project.metadata, }, }, } return this.executeGraphQLWithRetry(mutation, variables, 'createTenant', (data) => data?.createTenant?.id) } /** * Execute a GraphQL query/mutation with retry logic */ private async executeGraphQLWithRetry( query: string, variables: Record, operationName: string, extractId?: (data: T) => string | null | undefined ): Promise { const maxRetries = 3 let lastError: Error | null = null for (let attempt = 0; attempt <= maxRetries; attempt++) { if (attempt > 0) { const delay = Math.min(1000 * Math.pow(2, attempt - 1), 10000) // Exponential backoff, max 10s if (this.options.verbose) { console.log(`Retrying ${operationName} (attempt ${attempt + 1}/${maxRetries + 1}) after ${delay}ms...`) } await new Promise((resolve) => setTimeout(resolve, delay)) } try { const response = await this.executeGraphQL(query, variables, operationName) if (response) { if (extractId) { const id = extractId(response) return id || null } return (response as { id?: string })?.id || null } return null } catch (error) { lastError = error instanceof Error ? error : new Error(String(error)) // Don't retry on validation or authentication errors if (error instanceof Error) { if (error.message.includes('UNAUTHENTICATED') || error.message.includes('FORBIDDEN')) { throw error } if (error.message.includes('VALIDATION_ERROR') || error.message.includes('BAD_USER_INPUT')) { throw error } } if (attempt < maxRetries) { continue } } } throw lastError || new Error(`Failed to execute ${operationName} after ${maxRetries + 1} attempts`) } /** * Execute a GraphQL query/mutation */ private async executeGraphQL( query: string, variables: Record, operationName: string ): Promise { if (this.options.dryRun) { if (this.options.verbose) { console.log(`[DRY RUN] Would execute: ${operationName}`) console.log(` Variables:`, JSON.stringify(variables, null, 2)) } return null } const headers: Record = { 'Content-Type': 'application/json', } if (this.authToken) { headers['Authorization'] = `Bearer ${this.authToken}` } try { const response = await fetch(this.options.apiUrl, { method: 'POST', headers, body: JSON.stringify({ query, variables, operationName, }), }) if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`) } const result: GraphQLResponse = await response.json() if (result.errors) { const errorMessages = result.errors.map((e) => e.message).join(', ') const errorCodes = result.errors.map((e) => e.extensions?.code).filter(Boolean).join(', ') throw new Error(`${errorCodes ? `[${errorCodes}] ` : ''}${errorMessages}`) } return result.data || null } catch (error) { if (error instanceof Error) { throw error } throw new Error(`Failed to execute ${operationName}: ${String(error)}`) } } /** * Import all scanned projects */ async importProjects(projects: ProjectInfo[]): Promise { const result: ScanResult = { created: [], skipped: [], errors: [], } if (projects.length === 0) { console.log('No projects found to import.') return result } if (!this.options.outputFormat || this.options.outputFormat === 'text') { console.log(`\nFound ${projects.length} projects to import:`) projects.forEach((p) => { console.log(` - ${p.name} (${p.type})`) }) } if (this.options.dryRun) { if (!this.options.outputFormat || this.options.outputFormat === 'text') { console.log('\n[DRY RUN] Would create the following tenants:') } } else { if (!this.options.outputFormat || this.options.outputFormat === 'text') { console.log(`\nCreating tenants...`) } } // Create progress bar const progressBar = this.options.outputFormat !== 'json' && !this.options.verbose ? new cliProgress.SingleBar({ format: 'Progress |{bar}| {percentage}% | {value}/{total} projects | ETA: {eta}s', barCompleteChar: '\u2588', barIncompleteChar: '\u2591', hideCursor: true, }) : null if (progressBar) { progressBar.start(projects.length, 0) } // Process projects in batches const batchSize = this.options.batchSize || 1 for (let i = 0; i < projects.length; i += batchSize) { const batch = projects.slice(i, i + batchSize) const batchPromises = batch.map(async (project) => { try { // Check for duplicates if (this.options.skipExisting) { const existing = this.isDuplicate(project) if (existing) { result.skipped.push({ name: project.name, reason: `Already exists as tenant: ${existing.name} (ID: ${existing.id})`, }) if (progressBar) progressBar.increment() return } } const id = await this.createTenant(project) if (id) { result.created.push({ name: project.name, id }) if (this.options.verbose && this.options.outputFormat !== 'json') { console.log(`✓ Created tenant for ${project.name} (ID: ${id})`) } } else if (!this.options.dryRun) { result.errors.push({ name: project.name, error: 'Failed to create tenant (no ID returned)', }) } } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error) result.errors.push({ name: project.name, error: errorMessage, }) if (this.options.verbose && this.options.outputFormat !== 'json') { console.error(`✗ Error processing ${project.name}: ${errorMessage}`) } } finally { if (progressBar) progressBar.increment() } }) await Promise.all(batchPromises) } if (progressBar) { progressBar.stop() } // Print summary if (this.options.outputFormat === 'json') { console.log(JSON.stringify(result, null, 2)) } else { console.log(`\nSummary:`) console.log(` Created: ${result.created.length}`) console.log(` Skipped: ${result.skipped.length}`) console.log(` Errors: ${result.errors.length}`) console.log(` Total: ${projects.length}`) if (result.errors.length > 0 && this.options.verbose) { console.log(`\nErrors:`) result.errors.forEach((e) => { console.log(` - ${e.name}: ${e.error}`) }) } } return result } } // Main execution async function main() { program .name('scan-projects') .description('Scan projects directory and create Tenants in Sankofa system') .option('--dry-run', 'Show what would be created without actually creating', false) .option('--skip-existing', 'Skip projects that already exist as tenants', false) .option('--projects-dir ', 'Directory to scan', process.env.PROJECTS_DIR || join(process.env.HOME || '~', 'projects')) .option('--api-url ', 'GraphQL API URL', process.env.GRAPHQL_API_URL || 'http://localhost:4000/graphql') .option('--email ', 'Email for authentication', process.env.GRAPHQL_EMAIL) .option('--password ', 'Password for authentication', process.env.GRAPHQL_PASSWORD) .option('--auth-token ', 'Direct authentication token', process.env.GRAPHQL_AUTH_TOKEN) .option('--output ', 'Output format: text or json', 'text') .option('--verbose', 'Enable verbose logging', false) .option('--batch-size ', 'Number of projects to process in parallel', '1') .parse(process.argv) const rawOptions = program.opts>() // Map commander options to ScanOptions const options: ScanOptions = { projectsDir: (rawOptions.projectsDir as string) || process.env.PROJECTS_DIR || join(process.env.HOME || '~', 'projects'), dryRun: rawOptions.dryRun === true, skipExisting: rawOptions.skipExisting === true, apiUrl: (rawOptions.apiUrl as string) || process.env.GRAPHQL_API_URL || 'http://localhost:4000/graphql', email: (rawOptions.email as string) || process.env.GRAPHQL_EMAIL, password: (rawOptions.password as string) || process.env.GRAPHQL_PASSWORD, authToken: (rawOptions.authToken as string) || process.env.GRAPHQL_AUTH_TOKEN, outputFormat: ((rawOptions.output as string) || 'text') as 'text' | 'json', verbose: rawOptions.verbose === true, batchSize: 1, } // Validate batch size const batchSize = parseInt((rawOptions.batchSize as string) || '1', 10) if (isNaN(batchSize) || batchSize < 1) { console.error('Error: --batch-size must be a positive integer') process.exit(1) } options.batchSize = batchSize // Validate output format if (options.outputFormat !== 'text' && options.outputFormat !== 'json') { console.error('Error: --output must be "text" or "json"') process.exit(1) } const scanner = new ProjectScanner(options) try { // Authenticate (skip in dry-run mode) if (!options.dryRun) { await scanner.authenticate() // Load existing tenants for duplicate detection await scanner.loadExistingTenants() } else if (options.verbose) { console.log('[DRY RUN] Skipping authentication') } // Scan projects const projects = await scanner.scanAllProjects() // Import projects await scanner.importProjects(projects) } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error) console.error('Fatal error:', errorMessage) if (options.verbose && error instanceof Error && error.stack) { console.error(error.stack) } process.exit(1) } } if (require.main === module) { main() } export { ProjectScanner, ProjectInfo, ScanResult, ScanOptions }