Add generic CSV import tool for bulk pet ingestion

Two-step flow: upload CSV, auto-detect and map columns, preview first 5 rows, then import. Auto-detects common column names (name, species, breed, age, sex, etc.). Generates unique slugs from pet names, handles duplicates within the batch and against existing records. Reports imported count, skipped rows, and per-row errors. Requires 'import' permission. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-03-26 12:54:06 -05:00
parent 8e24f5637d
commit e7681f0b49
2 changed files with 554 additions and 0 deletions
@@ -0,0 +1,282 @@
+import type { PageServerLoad, Actions } from './$types';
+import { fail, error } from '@sveltejs/kit';
+import { db } from '$lib/server/db';
+import { pets } from '$lib/server/schema';
+import { slugify } from '$lib/slugify';
+import { hasPermission } from '$lib/roles';
+import type { Role } from '$lib/roles';
+import { logAudit } from '$lib/server/audit';
+
+// Pet fields that can be mapped from CSV
+export const PET_FIELDS = [
+	{ key: 'name', label: 'Name', required: true },
+	{ key: 'species', label: 'Species (dog/cat/other)' },
+	{ key: 'breed', label: 'Breed' },
+	{ key: 'age', label: 'Age' },
+	{ key: 'sex', label: 'Sex (male/female/unknown)' },
+	{ key: 'color', label: 'Color' },
+	{ key: 'weight', label: 'Weight' },
+	{ key: 'status', label: 'Status' },
+	{ key: 'description', label: 'Description' },
+	{ key: 'medicalNotes', label: 'Medical Notes' },
+	{ key: 'behavioralNotes', label: 'Behavioral Notes' },
+	{ key: 'specialNeeds', label: 'Special Needs' },
+	{ key: 'microchipId', label: 'Microchip ID' },
+	{ key: 'adoptionFee', label: 'Adoption Fee' },
+	{ key: 'skip', label: '— Skip this column —' }
+] as const;
+
+type PetFieldKey = (typeof PET_FIELDS)[number]['key'];
+
+function parseCSV(text: string): string[][] {
+	const rows: string[][] = [];
+	const lines = text.split(/\r?\n/);
+
+	for (const line of lines) {
+		if (!line.trim()) continue;
+
+		const cells: string[] = [];
+		let inQuotes = false;
+		let cell = '';
+
+		for (let i = 0; i < line.length; i++) {
+			const ch = line[i];
+
+			if (ch === '"') {
+				if (inQuotes && line[i + 1] === '"') {
+					cell += '"';
+					i++; // skip next quote
+				} else {
+					inQuotes = !inQuotes;
+				}
+			} else if (ch === ',' && !inQuotes) {
+				cells.push(cell.trim());
+				cell = '';
+			} else {
+				cell += ch;
+			}
+		}
+		cells.push(cell.trim());
+		rows.push(cells);
+	}
+
+	return rows;
+}
+
+function normalizeSpecies(val: string): 'dog' | 'cat' | 'other' {
+	const v = val.toLowerCase().trim();
+	if (v === 'dog' || v === 'canine') return 'dog';
+	if (v === 'cat' || v === 'feline') return 'cat';
+	return 'other';
+}
+
+function normalizeSex(val: string): 'male' | 'female' | 'unknown' {
+	const v = val.toLowerCase().trim();
+	if (v === 'male' || v === 'm') return 'male';
+	if (v === 'female' || v === 'f') return 'female';
+	return 'unknown';
+}
+
+function normalizeStatus(val: string): string {
+	const v = val.toLowerCase().trim();
+	const validStatuses = [
+		'available', 'adopted', 'foster', 'medical_hold', 'behavioral_hold',
+		'pending', 'trial', 'hospice', 'nursing', 'whelping', 'deceased', 'transferred'
+	];
+	return validStatuses.includes(v) ? v : 'available';
+}
+
+export const load: PageServerLoad = async ({ locals }) => {
+	if (!hasPermission(locals.user!.role as Role, 'import')) {
+		error(403, 'Access denied: import permission required.');
+	}
+
+	return {};
+};
+
+export const actions: Actions = {
+	// Parse uploaded CSV and return preview data
+	parse: async ({ request, locals }) => {
+		if (!hasPermission(locals.user!.role as Role, 'import')) {
+			return fail(403, { error: 'Access denied.' });
+		}
+
+		const fd = await request.formData();
+		const file = fd.get('csv') as File | null;
+
+		if (!file || file.size === 0) {
+			return fail(400, { error: 'Please upload a CSV file.' });
+		}
+
+		if (!file.name.endsWith('.csv') && file.type !== 'text/csv') {
+			return fail(400, { error: 'File must be a CSV.' });
+		}
+
+		if (file.size > 5 * 1024 * 1024) {
+			return fail(400, { error: 'File too large (max 5MB).' });
+		}
+
+		const text = await file.text();
+		const rows = parseCSV(text);
+
+		if (rows.length < 2) {
+			return fail(400, { error: 'CSV must have a header row and at least one data row.' });
+		}
+
+		const headers = rows[0];
+		const previewRows = rows.slice(1, 6); // First 5 data rows
+		const totalRows = rows.length - 1;
+
+		// Auto-detect column mappings based on header names
+		const autoMappings: Record<number, PetFieldKey> = {};
+		for (let i = 0; i < headers.length; i++) {
+			const h = headers[i].toLowerCase().trim();
+			if (h === 'name' || h === 'pet name' || h === 'animal name') autoMappings[i] = 'name';
+			else if (h === 'species' || h === 'type' || h === 'animal type') autoMappings[i] = 'species';
+			else if (h === 'breed') autoMappings[i] = 'breed';
+			else if (h === 'age' || h === 'age estimate') autoMappings[i] = 'age';
+			else if (h === 'sex' || h === 'gender') autoMappings[i] = 'sex';
+			else if (h === 'color' || h === 'colour') autoMappings[i] = 'color';
+			else if (h === 'weight') autoMappings[i] = 'weight';
+			else if (h === 'status') autoMappings[i] = 'status';
+			else if (h === 'description' || h === 'bio' || h === 'about') autoMappings[i] = 'description';
+			else if (h === 'medical notes' || h === 'medical') autoMappings[i] = 'medicalNotes';
+			else if (h === 'behavioral notes' || h === 'behavior' || h === 'behavioural') autoMappings[i] = 'behavioralNotes';
+			else if (h === 'special needs') autoMappings[i] = 'specialNeeds';
+			else if (h === 'microchip' || h === 'microchip id' || h === 'chip') autoMappings[i] = 'microchipId';
+			else if (h === 'adoption fee' || h === 'fee') autoMappings[i] = 'adoptionFee';
+			else autoMappings[i] = 'skip';
+		}
+
+		// Store the CSV data as a JSON string so we can pass it in the next form step
+		const csvData = JSON.stringify({ headers, rows: rows.slice(1) });
+
+		return {
+			parsed: true,
+			headers,
+			previewRows,
+			totalRows,
+			autoMappings,
+			csvData,
+			petFields: PET_FIELDS
+		};
+	},
+
+	// Import mapped CSV data into the pets table
+	import: async ({ request, locals }) => {
+		if (!hasPermission(locals.user!.role as Role, 'import')) {
+			return fail(403, { error: 'Access denied.' });
+		}
+
+		const fd = await request.formData();
+		const csvDataStr = fd.get('csvData')?.toString() ?? '';
+
+		if (!csvDataStr) {
+			return fail(400, { error: 'CSV data missing. Please re-upload.' });
+		}
+
+		let csvData: { headers: string[]; rows: string[][] };
+		try {
+			csvData = JSON.parse(csvDataStr);
+		} catch {
+			return fail(400, { error: 'Invalid CSV data.' });
+		}
+
+		const { headers, rows } = csvData;
+
+		// Build column mapping from form: mapping_0, mapping_1, ...
+		const columnMap: Record<number, PetFieldKey> = {};
+		for (let i = 0; i < headers.length; i++) {
+			const val = fd.get(`mapping_${i}`)?.toString() as PetFieldKey | undefined;
+			if (val) columnMap[i] = val;
+		}
+
+		// Verify 'name' is mapped
+		const hasName = Object.values(columnMap).includes('name');
+		if (!hasName) {
+			return fail(400, { error: 'You must map a column to "Name".' });
+		}
+
+		let imported = 0;
+		let skipped = 0;
+		const errors: string[] = [];
+
+		// Track slugs within this batch to avoid duplicates
+		const batchSlugs = new Set<string>();
+
+		// Get existing slugs from db
+		const existingSlugRows = await db.select({ slug: pets.slug }).from(pets);
+		const existingSlugs = new Set(existingSlugRows.map(r => r.slug));
+
+		for (let rowIdx = 0; rowIdx < rows.length; rowIdx++) {
+			const row = rows[rowIdx];
+			const rowNum = rowIdx + 2; // 1-indexed, +1 for header
+
+			// Extract mapped values
+			const mapped: Record<string, string> = {};
+			for (let colIdx = 0; colIdx < headers.length; colIdx++) {
+				const field = columnMap[colIdx];
+				if (field && field !== 'skip' && row[colIdx] !== undefined) {
+					mapped[field] = row[colIdx].trim();
+				}
+			}
+
+			const name = mapped['name'];
+			if (!name) {
+				errors.push(`Row ${rowNum}: skipped — name is empty.`);
+				skipped++;
+				continue;
+			}
+
+			// Generate unique slug
+			let baseSlug = slugify(name);
+			let slug = baseSlug;
+			let counter = 1;
+			while (existingSlugs.has(slug) || batchSlugs.has(slug)) {
+				slug = `${baseSlug}-${counter}`;
+				counter++;
+			}
+			batchSlugs.add(slug);
+			existingSlugs.add(slug);
+
+			try {
+				await db.insert(pets).values({
+					name,
+					slug,
+					species: mapped['species'] ? normalizeSpecies(mapped['species']) : 'dog',
+					breed: mapped['breed'] || null,
+					age: mapped['age'] || null,
+					sex: mapped['sex'] ? normalizeSex(mapped['sex']) : 'unknown',
+					color: mapped['color'] || null,
+					weight: mapped['weight'] || null,
+					status: mapped['status'] ? normalizeStatus(mapped['status']) as any : 'available',
+					description: mapped['description'] || null,
+					medicalNotes: mapped['medicalNotes'] || null,
+					behavioralNotes: mapped['behavioralNotes'] || null,
+					specialNeeds: mapped['specialNeeds'] || null,
+					microchipId: mapped['microchipId'] || null,
+					adoptionFee: mapped['adoptionFee'] ? parseInt(mapped['adoptionFee'], 10) || null : null,
+					published: true
+				});
+				imported++;
+			} catch (err) {
+				errors.push(`Row ${rowNum} (${name}): ${err instanceof Error ? err.message : 'Insert failed.'}`);
+				skipped++;
+			}
+		}
+
+		await logAudit({
+			userId: locals.user!.id,
+			action: 'csv_import',
+			entity: 'pets',
+			diff: { imported, skipped, errorCount: errors.length }
+		});
+
+		return {
+			importComplete: true,
+			imported,
+			skipped,
+			errors: errors.slice(0, 20) // Cap error output
+		};
+	}
+};