Add generic CSV import tool for bulk pet ingestion

Two-step flow: upload CSV, auto-detect and map columns, preview first 5
rows, then import. Auto-detects common column names (name, species,
breed, age, sex, etc.). Generates unique slugs from pet names, handles
duplicates within the batch and against existing records. Reports
imported count, skipped rows, and per-row errors. Requires 'import'
permission.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-26 12:54:06 -05:00
co-authored by Claude Sonnet 4.6
parent 8e24f5637d
commit e7681f0b49
2 changed files with 554 additions and 0 deletions
+282
View File
@@ -0,0 +1,282 @@
import type { PageServerLoad, Actions } from './$types';
import { fail, error } from '@sveltejs/kit';
import { db } from '$lib/server/db';
import { pets } from '$lib/server/schema';
import { slugify } from '$lib/slugify';
import { hasPermission } from '$lib/roles';
import type { Role } from '$lib/roles';
import { logAudit } from '$lib/server/audit';
// Pet fields that can be mapped from CSV
export const PET_FIELDS = [
{ key: 'name', label: 'Name', required: true },
{ key: 'species', label: 'Species (dog/cat/other)' },
{ key: 'breed', label: 'Breed' },
{ key: 'age', label: 'Age' },
{ key: 'sex', label: 'Sex (male/female/unknown)' },
{ key: 'color', label: 'Color' },
{ key: 'weight', label: 'Weight' },
{ key: 'status', label: 'Status' },
{ key: 'description', label: 'Description' },
{ key: 'medicalNotes', label: 'Medical Notes' },
{ key: 'behavioralNotes', label: 'Behavioral Notes' },
{ key: 'specialNeeds', label: 'Special Needs' },
{ key: 'microchipId', label: 'Microchip ID' },
{ key: 'adoptionFee', label: 'Adoption Fee' },
{ key: 'skip', label: '— Skip this column —' }
] as const;
type PetFieldKey = (typeof PET_FIELDS)[number]['key'];
function parseCSV(text: string): string[][] {
const rows: string[][] = [];
const lines = text.split(/\r?\n/);
for (const line of lines) {
if (!line.trim()) continue;
const cells: string[] = [];
let inQuotes = false;
let cell = '';
for (let i = 0; i < line.length; i++) {
const ch = line[i];
if (ch === '"') {
if (inQuotes && line[i + 1] === '"') {
cell += '"';
i++; // skip next quote
} else {
inQuotes = !inQuotes;
}
} else if (ch === ',' && !inQuotes) {
cells.push(cell.trim());
cell = '';
} else {
cell += ch;
}
}
cells.push(cell.trim());
rows.push(cells);
}
return rows;
}
function normalizeSpecies(val: string): 'dog' | 'cat' | 'other' {
const v = val.toLowerCase().trim();
if (v === 'dog' || v === 'canine') return 'dog';
if (v === 'cat' || v === 'feline') return 'cat';
return 'other';
}
function normalizeSex(val: string): 'male' | 'female' | 'unknown' {
const v = val.toLowerCase().trim();
if (v === 'male' || v === 'm') return 'male';
if (v === 'female' || v === 'f') return 'female';
return 'unknown';
}
function normalizeStatus(val: string): string {
const v = val.toLowerCase().trim();
const validStatuses = [
'available', 'adopted', 'foster', 'medical_hold', 'behavioral_hold',
'pending', 'trial', 'hospice', 'nursing', 'whelping', 'deceased', 'transferred'
];
return validStatuses.includes(v) ? v : 'available';
}
export const load: PageServerLoad = async ({ locals }) => {
if (!hasPermission(locals.user!.role as Role, 'import')) {
error(403, 'Access denied: import permission required.');
}
return {};
};
export const actions: Actions = {
// Parse uploaded CSV and return preview data
parse: async ({ request, locals }) => {
if (!hasPermission(locals.user!.role as Role, 'import')) {
return fail(403, { error: 'Access denied.' });
}
const fd = await request.formData();
const file = fd.get('csv') as File | null;
if (!file || file.size === 0) {
return fail(400, { error: 'Please upload a CSV file.' });
}
if (!file.name.endsWith('.csv') && file.type !== 'text/csv') {
return fail(400, { error: 'File must be a CSV.' });
}
if (file.size > 5 * 1024 * 1024) {
return fail(400, { error: 'File too large (max 5MB).' });
}
const text = await file.text();
const rows = parseCSV(text);
if (rows.length < 2) {
return fail(400, { error: 'CSV must have a header row and at least one data row.' });
}
const headers = rows[0];
const previewRows = rows.slice(1, 6); // First 5 data rows
const totalRows = rows.length - 1;
// Auto-detect column mappings based on header names
const autoMappings: Record<number, PetFieldKey> = {};
for (let i = 0; i < headers.length; i++) {
const h = headers[i].toLowerCase().trim();
if (h === 'name' || h === 'pet name' || h === 'animal name') autoMappings[i] = 'name';
else if (h === 'species' || h === 'type' || h === 'animal type') autoMappings[i] = 'species';
else if (h === 'breed') autoMappings[i] = 'breed';
else if (h === 'age' || h === 'age estimate') autoMappings[i] = 'age';
else if (h === 'sex' || h === 'gender') autoMappings[i] = 'sex';
else if (h === 'color' || h === 'colour') autoMappings[i] = 'color';
else if (h === 'weight') autoMappings[i] = 'weight';
else if (h === 'status') autoMappings[i] = 'status';
else if (h === 'description' || h === 'bio' || h === 'about') autoMappings[i] = 'description';
else if (h === 'medical notes' || h === 'medical') autoMappings[i] = 'medicalNotes';
else if (h === 'behavioral notes' || h === 'behavior' || h === 'behavioural') autoMappings[i] = 'behavioralNotes';
else if (h === 'special needs') autoMappings[i] = 'specialNeeds';
else if (h === 'microchip' || h === 'microchip id' || h === 'chip') autoMappings[i] = 'microchipId';
else if (h === 'adoption fee' || h === 'fee') autoMappings[i] = 'adoptionFee';
else autoMappings[i] = 'skip';
}
// Store the CSV data as a JSON string so we can pass it in the next form step
const csvData = JSON.stringify({ headers, rows: rows.slice(1) });
return {
parsed: true,
headers,
previewRows,
totalRows,
autoMappings,
csvData,
petFields: PET_FIELDS
};
},
// Import mapped CSV data into the pets table
import: async ({ request, locals }) => {
if (!hasPermission(locals.user!.role as Role, 'import')) {
return fail(403, { error: 'Access denied.' });
}
const fd = await request.formData();
const csvDataStr = fd.get('csvData')?.toString() ?? '';
if (!csvDataStr) {
return fail(400, { error: 'CSV data missing. Please re-upload.' });
}
let csvData: { headers: string[]; rows: string[][] };
try {
csvData = JSON.parse(csvDataStr);
} catch {
return fail(400, { error: 'Invalid CSV data.' });
}
const { headers, rows } = csvData;
// Build column mapping from form: mapping_0, mapping_1, ...
const columnMap: Record<number, PetFieldKey> = {};
for (let i = 0; i < headers.length; i++) {
const val = fd.get(`mapping_${i}`)?.toString() as PetFieldKey | undefined;
if (val) columnMap[i] = val;
}
// Verify 'name' is mapped
const hasName = Object.values(columnMap).includes('name');
if (!hasName) {
return fail(400, { error: 'You must map a column to "Name".' });
}
let imported = 0;
let skipped = 0;
const errors: string[] = [];
// Track slugs within this batch to avoid duplicates
const batchSlugs = new Set<string>();
// Get existing slugs from db
const existingSlugRows = await db.select({ slug: pets.slug }).from(pets);
const existingSlugs = new Set(existingSlugRows.map(r => r.slug));
for (let rowIdx = 0; rowIdx < rows.length; rowIdx++) {
const row = rows[rowIdx];
const rowNum = rowIdx + 2; // 1-indexed, +1 for header
// Extract mapped values
const mapped: Record<string, string> = {};
for (let colIdx = 0; colIdx < headers.length; colIdx++) {
const field = columnMap[colIdx];
if (field && field !== 'skip' && row[colIdx] !== undefined) {
mapped[field] = row[colIdx].trim();
}
}
const name = mapped['name'];
if (!name) {
errors.push(`Row ${rowNum}: skipped — name is empty.`);
skipped++;
continue;
}
// Generate unique slug
let baseSlug = slugify(name);
let slug = baseSlug;
let counter = 1;
while (existingSlugs.has(slug) || batchSlugs.has(slug)) {
slug = `${baseSlug}-${counter}`;
counter++;
}
batchSlugs.add(slug);
existingSlugs.add(slug);
try {
await db.insert(pets).values({
name,
slug,
species: mapped['species'] ? normalizeSpecies(mapped['species']) : 'dog',
breed: mapped['breed'] || null,
age: mapped['age'] || null,
sex: mapped['sex'] ? normalizeSex(mapped['sex']) : 'unknown',
color: mapped['color'] || null,
weight: mapped['weight'] || null,
status: mapped['status'] ? normalizeStatus(mapped['status']) as any : 'available',
description: mapped['description'] || null,
medicalNotes: mapped['medicalNotes'] || null,
behavioralNotes: mapped['behavioralNotes'] || null,
specialNeeds: mapped['specialNeeds'] || null,
microchipId: mapped['microchipId'] || null,
adoptionFee: mapped['adoptionFee'] ? parseInt(mapped['adoptionFee'], 10) || null : null,
published: true
});
imported++;
} catch (err) {
errors.push(`Row ${rowNum} (${name}): ${err instanceof Error ? err.message : 'Insert failed.'}`);
skipped++;
}
}
await logAudit({
userId: locals.user!.id,
action: 'csv_import',
entity: 'pets',
diff: { imported, skipped, errorCount: errors.length }
});
return {
importComplete: true,
imported,
skipped,
errors: errors.slice(0, 20) // Cap error output
};
}
};
+272
View File
@@ -0,0 +1,272 @@
<script lang="ts">
import { enhance } from '$app/forms';
let { form } = $props();
let uploading = $state(false);
let importing = $state(false);
let dragOver = $state(false);
// Column mappings (col index -> field key)
let mappings = $state<Record<number, string>>({});
$effect(() => {
if (form?.autoMappings) {
mappings = { ...form.autoMappings };
}
});
function handleUploadEnhance() {
uploading = true;
return async ({ update }: { update: (opts?: { reset?: boolean }) => Promise<void> }) => {
uploading = false;
await update({ reset: false });
};
}
function handleImportEnhance() {
importing = true;
return async ({ update }: { update: (opts?: { reset?: boolean }) => Promise<void> }) => {
importing = false;
await update({ reset: false });
};
}
function reset() {
mappings = {};
// Reset form by navigating to same page
window.location.href = '/admin/import';
}
const petFieldOptions = [
{ key: 'skip', label: '— Skip —' },
{ key: 'name', label: 'Name *' },
{ key: 'species', label: 'Species' },
{ key: 'breed', label: 'Breed' },
{ key: 'age', label: 'Age' },
{ key: 'sex', label: 'Sex' },
{ key: 'color', label: 'Color' },
{ key: 'weight', label: 'Weight' },
{ key: 'status', label: 'Status' },
{ key: 'description', label: 'Description' },
{ key: 'medicalNotes', label: 'Medical Notes' },
{ key: 'behavioralNotes', label: 'Behavioral Notes' },
{ key: 'specialNeeds', label: 'Special Needs' },
{ key: 'microchipId', label: 'Microchip ID' },
{ key: 'adoptionFee', label: 'Adoption Fee' }
];
const inputClass = 'w-full px-3 py-2.5 rounded-lg border border-gray-200 bg-white text-gray-900 placeholder-gray-400 focus:outline-none focus:ring-2 focus:ring-teal-500 focus:border-teal-500 text-sm';
const selectClass = inputClass;
</script>
<svelte:head>
<title>CSV Import — Admin</title>
</svelte:head>
<div class="mb-6">
<h1 class="text-2xl font-bold text-gray-900">CSV Import</h1>
<p class="text-gray-500 mt-1">Bulk import pet records from a CSV file.</p>
</div>
<!-- Import complete results -->
{#if form?.importComplete}
<div class="mb-6 p-6 bg-white rounded-xl shadow-sm border border-gray-200">
<h2 class="text-lg font-semibold text-gray-900 mb-4">Import Complete</h2>
<div class="grid grid-cols-2 sm:grid-cols-3 gap-4 mb-4">
<div class="text-center p-4 bg-green-50 rounded-lg">
<p class="text-3xl font-bold text-green-700">{form.imported}</p>
<p class="text-sm text-green-600 mt-1">Imported</p>
</div>
<div class="text-center p-4 bg-red-50 rounded-lg">
<p class="text-3xl font-bold text-red-700">{form.skipped}</p>
<p class="text-sm text-red-600 mt-1">Skipped / Errors</p>
</div>
</div>
{#if form.errors && form.errors.length > 0}
<div class="mt-4">
<p class="text-sm font-medium text-gray-700 mb-2">Issues ({form.errors.length}):</p>
<ul class="text-xs text-red-700 bg-red-50 rounded-lg p-3 space-y-1 max-h-48 overflow-y-auto">
{#each form.errors as err}
<li>{err}</li>
{/each}
</ul>
</div>
{/if}
<div class="mt-6 flex gap-3">
<a
href="/admin/pets"
class="px-4 py-2 text-sm font-medium rounded-lg text-white bg-teal-600 hover:bg-teal-700 transition-colors"
>
View Pets
</a>
<button
type="button"
onclick={reset}
class="px-4 py-2 text-sm font-medium rounded-lg border border-gray-200 text-gray-700 hover:bg-gray-50 transition-colors cursor-pointer"
>
Import Another File
</button>
</div>
</div>
<!-- Column mapping + preview -->
{:else if form?.parsed}
<div class="mb-6 p-6 bg-white rounded-xl shadow-sm border border-gray-200">
<div class="flex items-center justify-between mb-4">
<h2 class="text-lg font-semibold text-gray-900">Map Columns</h2>
<span class="text-sm text-gray-500">{form.totalRows} row{form.totalRows !== 1 ? 's' : ''} detected</span>
</div>
<form method="POST" action="?/import" use:enhance={handleImportEnhance}>
<input type="hidden" name="csvData" value={form.csvData} />
<!-- Column mapping table -->
<div class="overflow-x-auto mb-6">
<table class="w-full text-sm border-collapse">
<thead>
<tr class="border-b border-gray-200 bg-gray-50">
<th class="text-left px-3 py-2 font-medium text-gray-500 whitespace-nowrap w-48">CSV Column</th>
<th class="text-left px-3 py-2 font-medium text-gray-500 w-48">Map to Field</th>
<th class="text-left px-3 py-2 font-medium text-gray-500">Preview (first 5 rows)</th>
</tr>
</thead>
<tbody class="divide-y divide-gray-100">
{#each form.headers as header, colIdx}
<tr class="hover:bg-gray-50">
<td class="px-3 py-2 font-medium text-gray-900 whitespace-nowrap">
{header}
</td>
<td class="px-3 py-2">
<select
name="mapping_{colIdx}"
bind:value={mappings[colIdx]}
class="{selectClass} w-44"
>
{#each petFieldOptions as opt}
<option value={opt.key}>{opt.label}</option>
{/each}
</select>
</td>
<td class="px-3 py-2">
<div class="flex flex-wrap gap-1">
{#each form.previewRows as previewRow}
{#if previewRow[colIdx] !== undefined}
<span class="inline-block text-xs bg-gray-100 text-gray-600 px-2 py-0.5 rounded max-w-[120px] truncate" title={previewRow[colIdx]}>
{previewRow[colIdx] || '—'}
</span>
{/if}
{/each}
</div>
</td>
</tr>
{/each}
</tbody>
</table>
</div>
<div class="p-3 bg-yellow-50 border border-yellow-200 rounded-lg mb-6 text-sm text-yellow-800">
<strong>Required:</strong> At least one column must be mapped to <strong>Name</strong>.
Species defaults to "dog" and sex defaults to "unknown" if not mapped.
</div>
<div class="flex gap-3">
<button
type="submit"
disabled={importing}
class="px-6 py-2.5 text-sm font-medium rounded-lg text-white bg-teal-600 hover:bg-teal-700 transition-colors disabled:opacity-50 cursor-pointer"
>
{importing ? 'Importing...' : `Import ${form.totalRows} Records`}
</button>
<button
type="button"
onclick={reset}
class="px-4 py-2 text-sm font-medium rounded-lg border border-gray-200 text-gray-700 hover:bg-gray-50 transition-colors cursor-pointer"
>
Cancel
</button>
</div>
</form>
</div>
<!-- Upload form -->
{:else}
{#if form?.error}
<div class="mb-6 p-4 bg-red-50 border border-red-200 rounded-xl text-sm text-red-700">
{form.error}
</div>
{/if}
<div class="bg-white rounded-xl shadow-sm border border-gray-200 p-6 mb-6">
<h2 class="text-lg font-semibold text-gray-900 mb-4">Upload CSV File</h2>
<form method="POST" action="?/parse" enctype="multipart/form-data" use:enhance={handleUploadEnhance}>
<div
class="border-2 border-dashed rounded-xl p-10 text-center transition-colors {dragOver ? 'border-teal-400 bg-teal-50' : 'border-gray-200 hover:border-gray-300'}"
ondragover={(e) => { e.preventDefault(); dragOver = true; }}
ondragleave={() => dragOver = false}
ondrop={(e) => { e.preventDefault(); dragOver = false; }}
role="region"
aria-label="File upload area"
>
<svg class="w-12 h-12 text-gray-300 mx-auto mb-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="1.5" d="M9 13h6m-3-3v6m5 5H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z" />
</svg>
<p class="text-gray-600 font-medium mb-1">Drop your CSV file here</p>
<p class="text-sm text-gray-400 mb-4">or click to browse</p>
<input
type="file"
id="csv"
name="csv"
accept=".csv,text/csv"
required
class="sr-only"
/>
<label
for="csv"
class="inline-flex items-center px-4 py-2 text-sm font-medium rounded-lg border border-gray-200 text-gray-700 hover:bg-gray-50 transition-colors cursor-pointer"
>
Choose File
</label>
</div>
<div class="mt-4 flex justify-end">
<button
type="submit"
disabled={uploading}
class="px-6 py-2.5 text-sm font-medium rounded-lg text-white bg-teal-600 hover:bg-teal-700 transition-colors disabled:opacity-50 cursor-pointer"
>
{uploading ? 'Parsing...' : 'Parse CSV'}
</button>
</div>
</form>
</div>
<!-- Instructions -->
<div class="bg-white rounded-xl shadow-sm border border-gray-200 p-6">
<h3 class="text-sm font-semibold text-gray-800 mb-3">How it works</h3>
<ol class="text-sm text-gray-600 space-y-2 list-decimal list-inside">
<li>Upload a CSV file with a header row and one pet per row.</li>
<li>The importer auto-detects common column names.</li>
<li>Review the preview and adjust column mappings as needed.</li>
<li>Click "Import" to insert all records. Slugs are generated automatically from pet names.</li>
</ol>
<div class="mt-4 p-3 bg-gray-50 rounded-lg">
<p class="text-xs font-medium text-gray-700 mb-1">Supported columns:</p>
<p class="text-xs text-gray-500">name, species, breed, age, sex, color, weight, status, description, medical notes, behavioral notes, special needs, microchip ID, adoption fee</p>
</div>
<div class="mt-3 p-3 bg-gray-50 rounded-lg">
<p class="text-xs font-medium text-gray-700 mb-1">Valid values:</p>
<ul class="text-xs text-gray-500 space-y-0.5">
<li><strong>species:</strong> dog, cat, other</li>
<li><strong>sex:</strong> male, female, unknown</li>
<li><strong>status:</strong> available, adopted, foster, medical_hold, behavioral_hold, pending, trial, hospice, nursing, whelping, deceased, transferred</li>
</ul>
</div>
</div>
{/if}