From e7681f0b49322a5a07685dc0b4ca040a2dab97a7 Mon Sep 17 00:00:00 2001 From: Justin Reiners Date: Thu, 26 Mar 2026 12:54:06 -0500 Subject: [PATCH] Add generic CSV import tool for bulk pet ingestion Two-step flow: upload CSV, auto-detect and map columns, preview first 5 rows, then import. Auto-detects common column names (name, species, breed, age, sex, etc.). Generates unique slugs from pet names, handles duplicates within the batch and against existing records. Reports imported count, skipped rows, and per-row errors. Requires 'import' permission. Co-Authored-By: Claude Sonnet 4.6 --- src/routes/admin/import/+page.server.ts | 282 ++++++++++++++++++++++++ src/routes/admin/import/+page.svelte | 272 +++++++++++++++++++++++ 2 files changed, 554 insertions(+) create mode 100644 src/routes/admin/import/+page.server.ts create mode 100644 src/routes/admin/import/+page.svelte diff --git a/src/routes/admin/import/+page.server.ts b/src/routes/admin/import/+page.server.ts new file mode 100644 index 0000000..0dbfdcb --- /dev/null +++ b/src/routes/admin/import/+page.server.ts @@ -0,0 +1,282 @@ +import type { PageServerLoad, Actions } from './$types'; +import { fail, error } from '@sveltejs/kit'; +import { db } from '$lib/server/db'; +import { pets } from '$lib/server/schema'; +import { slugify } from '$lib/slugify'; +import { hasPermission } from '$lib/roles'; +import type { Role } from '$lib/roles'; +import { logAudit } from '$lib/server/audit'; + +// Pet fields that can be mapped from CSV +export const PET_FIELDS = [ + { key: 'name', label: 'Name', required: true }, + { key: 'species', label: 'Species (dog/cat/other)' }, + { key: 'breed', label: 'Breed' }, + { key: 'age', label: 'Age' }, + { key: 'sex', label: 'Sex (male/female/unknown)' }, + { key: 'color', label: 'Color' }, + { key: 'weight', label: 'Weight' }, + { key: 'status', label: 'Status' }, + { key: 'description', label: 'Description' }, + { key: 'medicalNotes', label: 'Medical Notes' }, + { key: 'behavioralNotes', label: 'Behavioral Notes' }, + { key: 'specialNeeds', label: 'Special Needs' }, + { key: 'microchipId', label: 'Microchip ID' }, + { key: 'adoptionFee', label: 'Adoption Fee' }, + { key: 'skip', label: '— Skip this column —' } +] as const; + +type PetFieldKey = (typeof PET_FIELDS)[number]['key']; + +function parseCSV(text: string): string[][] { + const rows: string[][] = []; + const lines = text.split(/\r?\n/); + + for (const line of lines) { + if (!line.trim()) continue; + + const cells: string[] = []; + let inQuotes = false; + let cell = ''; + + for (let i = 0; i < line.length; i++) { + const ch = line[i]; + + if (ch === '"') { + if (inQuotes && line[i + 1] === '"') { + cell += '"'; + i++; // skip next quote + } else { + inQuotes = !inQuotes; + } + } else if (ch === ',' && !inQuotes) { + cells.push(cell.trim()); + cell = ''; + } else { + cell += ch; + } + } + cells.push(cell.trim()); + rows.push(cells); + } + + return rows; +} + +function normalizeSpecies(val: string): 'dog' | 'cat' | 'other' { + const v = val.toLowerCase().trim(); + if (v === 'dog' || v === 'canine') return 'dog'; + if (v === 'cat' || v === 'feline') return 'cat'; + return 'other'; +} + +function normalizeSex(val: string): 'male' | 'female' | 'unknown' { + const v = val.toLowerCase().trim(); + if (v === 'male' || v === 'm') return 'male'; + if (v === 'female' || v === 'f') return 'female'; + return 'unknown'; +} + +function normalizeStatus(val: string): string { + const v = val.toLowerCase().trim(); + const validStatuses = [ + 'available', 'adopted', 'foster', 'medical_hold', 'behavioral_hold', + 'pending', 'trial', 'hospice', 'nursing', 'whelping', 'deceased', 'transferred' + ]; + return validStatuses.includes(v) ? v : 'available'; +} + +export const load: PageServerLoad = async ({ locals }) => { + if (!hasPermission(locals.user!.role as Role, 'import')) { + error(403, 'Access denied: import permission required.'); + } + + return {}; +}; + +export const actions: Actions = { + // Parse uploaded CSV and return preview data + parse: async ({ request, locals }) => { + if (!hasPermission(locals.user!.role as Role, 'import')) { + return fail(403, { error: 'Access denied.' }); + } + + const fd = await request.formData(); + const file = fd.get('csv') as File | null; + + if (!file || file.size === 0) { + return fail(400, { error: 'Please upload a CSV file.' }); + } + + if (!file.name.endsWith('.csv') && file.type !== 'text/csv') { + return fail(400, { error: 'File must be a CSV.' }); + } + + if (file.size > 5 * 1024 * 1024) { + return fail(400, { error: 'File too large (max 5MB).' }); + } + + const text = await file.text(); + const rows = parseCSV(text); + + if (rows.length < 2) { + return fail(400, { error: 'CSV must have a header row and at least one data row.' }); + } + + const headers = rows[0]; + const previewRows = rows.slice(1, 6); // First 5 data rows + const totalRows = rows.length - 1; + + // Auto-detect column mappings based on header names + const autoMappings: Record = {}; + for (let i = 0; i < headers.length; i++) { + const h = headers[i].toLowerCase().trim(); + if (h === 'name' || h === 'pet name' || h === 'animal name') autoMappings[i] = 'name'; + else if (h === 'species' || h === 'type' || h === 'animal type') autoMappings[i] = 'species'; + else if (h === 'breed') autoMappings[i] = 'breed'; + else if (h === 'age' || h === 'age estimate') autoMappings[i] = 'age'; + else if (h === 'sex' || h === 'gender') autoMappings[i] = 'sex'; + else if (h === 'color' || h === 'colour') autoMappings[i] = 'color'; + else if (h === 'weight') autoMappings[i] = 'weight'; + else if (h === 'status') autoMappings[i] = 'status'; + else if (h === 'description' || h === 'bio' || h === 'about') autoMappings[i] = 'description'; + else if (h === 'medical notes' || h === 'medical') autoMappings[i] = 'medicalNotes'; + else if (h === 'behavioral notes' || h === 'behavior' || h === 'behavioural') autoMappings[i] = 'behavioralNotes'; + else if (h === 'special needs') autoMappings[i] = 'specialNeeds'; + else if (h === 'microchip' || h === 'microchip id' || h === 'chip') autoMappings[i] = 'microchipId'; + else if (h === 'adoption fee' || h === 'fee') autoMappings[i] = 'adoptionFee'; + else autoMappings[i] = 'skip'; + } + + // Store the CSV data as a JSON string so we can pass it in the next form step + const csvData = JSON.stringify({ headers, rows: rows.slice(1) }); + + return { + parsed: true, + headers, + previewRows, + totalRows, + autoMappings, + csvData, + petFields: PET_FIELDS + }; + }, + + // Import mapped CSV data into the pets table + import: async ({ request, locals }) => { + if (!hasPermission(locals.user!.role as Role, 'import')) { + return fail(403, { error: 'Access denied.' }); + } + + const fd = await request.formData(); + const csvDataStr = fd.get('csvData')?.toString() ?? ''; + + if (!csvDataStr) { + return fail(400, { error: 'CSV data missing. Please re-upload.' }); + } + + let csvData: { headers: string[]; rows: string[][] }; + try { + csvData = JSON.parse(csvDataStr); + } catch { + return fail(400, { error: 'Invalid CSV data.' }); + } + + const { headers, rows } = csvData; + + // Build column mapping from form: mapping_0, mapping_1, ... + const columnMap: Record = {}; + for (let i = 0; i < headers.length; i++) { + const val = fd.get(`mapping_${i}`)?.toString() as PetFieldKey | undefined; + if (val) columnMap[i] = val; + } + + // Verify 'name' is mapped + const hasName = Object.values(columnMap).includes('name'); + if (!hasName) { + return fail(400, { error: 'You must map a column to "Name".' }); + } + + let imported = 0; + let skipped = 0; + const errors: string[] = []; + + // Track slugs within this batch to avoid duplicates + const batchSlugs = new Set(); + + // Get existing slugs from db + const existingSlugRows = await db.select({ slug: pets.slug }).from(pets); + const existingSlugs = new Set(existingSlugRows.map(r => r.slug)); + + for (let rowIdx = 0; rowIdx < rows.length; rowIdx++) { + const row = rows[rowIdx]; + const rowNum = rowIdx + 2; // 1-indexed, +1 for header + + // Extract mapped values + const mapped: Record = {}; + for (let colIdx = 0; colIdx < headers.length; colIdx++) { + const field = columnMap[colIdx]; + if (field && field !== 'skip' && row[colIdx] !== undefined) { + mapped[field] = row[colIdx].trim(); + } + } + + const name = mapped['name']; + if (!name) { + errors.push(`Row ${rowNum}: skipped — name is empty.`); + skipped++; + continue; + } + + // Generate unique slug + let baseSlug = slugify(name); + let slug = baseSlug; + let counter = 1; + while (existingSlugs.has(slug) || batchSlugs.has(slug)) { + slug = `${baseSlug}-${counter}`; + counter++; + } + batchSlugs.add(slug); + existingSlugs.add(slug); + + try { + await db.insert(pets).values({ + name, + slug, + species: mapped['species'] ? normalizeSpecies(mapped['species']) : 'dog', + breed: mapped['breed'] || null, + age: mapped['age'] || null, + sex: mapped['sex'] ? normalizeSex(mapped['sex']) : 'unknown', + color: mapped['color'] || null, + weight: mapped['weight'] || null, + status: mapped['status'] ? normalizeStatus(mapped['status']) as any : 'available', + description: mapped['description'] || null, + medicalNotes: mapped['medicalNotes'] || null, + behavioralNotes: mapped['behavioralNotes'] || null, + specialNeeds: mapped['specialNeeds'] || null, + microchipId: mapped['microchipId'] || null, + adoptionFee: mapped['adoptionFee'] ? parseInt(mapped['adoptionFee'], 10) || null : null, + published: true + }); + imported++; + } catch (err) { + errors.push(`Row ${rowNum} (${name}): ${err instanceof Error ? err.message : 'Insert failed.'}`); + skipped++; + } + } + + await logAudit({ + userId: locals.user!.id, + action: 'csv_import', + entity: 'pets', + diff: { imported, skipped, errorCount: errors.length } + }); + + return { + importComplete: true, + imported, + skipped, + errors: errors.slice(0, 20) // Cap error output + }; + } +}; diff --git a/src/routes/admin/import/+page.svelte b/src/routes/admin/import/+page.svelte new file mode 100644 index 0000000..e0686e6 --- /dev/null +++ b/src/routes/admin/import/+page.svelte @@ -0,0 +1,272 @@ + + + + CSV Import — Admin + + +
+

CSV Import

+

Bulk import pet records from a CSV file.

+
+ + +{#if form?.importComplete} +
+

Import Complete

+ +
+
+

{form.imported}

+

Imported

+
+
+

{form.skipped}

+

Skipped / Errors

+
+
+ + {#if form.errors && form.errors.length > 0} +
+

Issues ({form.errors.length}):

+
    + {#each form.errors as err} +
  • {err}
    • + {/each} +
+
+ {/if} + +
+ + View Pets + + +
+
+ + +{:else if form?.parsed} +
+
+

Map Columns

+ {form.totalRows} row{form.totalRows !== 1 ? 's' : ''} detected +
+ +
+ + + +
+ + + + + + + + + + {#each form.headers as header, colIdx} + + + + + + {/each} + +
CSV ColumnMap to FieldPreview (first 5 rows)
+ {header} + + + +
+ {#each form.previewRows as previewRow} + {#if previewRow[colIdx] !== undefined} + + {previewRow[colIdx] || '—'} + + {/if} + {/each} +
+
+
+ +
+ Required: At least one column must be mapped to Name. + Species defaults to "dog" and sex defaults to "unknown" if not mapped. +
+ +
+ + +
+
+
+ + +{:else} + {#if form?.error} +
+ {form.error} +
+ {/if} + +
+

Upload CSV File

+ +
+
{ e.preventDefault(); dragOver = true; }} + ondragleave={() => dragOver = false} + ondrop={(e) => { e.preventDefault(); dragOver = false; }} + role="region" + aria-label="File upload area" + > + + + +

Drop your CSV file here

+

or click to browse

+ + +
+ +
+ +
+
+
+ + +
+

How it works

+
    +
  1. Upload a CSV file with a header row and one pet per row.
    2. +
  2. The importer auto-detects common column names.
    3. +
  3. Review the preview and adjust column mappings as needed.
    4. +
  4. Click "Import" to insert all records. Slugs are generated automatically from pet names.
    5. +
+ +
+

Supported columns:

+

name, species, breed, age, sex, color, weight, status, description, medical notes, behavioral notes, special needs, microchip ID, adoption fee

+
+ +
+

Valid values:

+
    +
  • species: dog, cat, other
    • +
  • sex: male, female, unknown
    • +
  • status: available, adopted, foster, medical_hold, behavioral_hold, pending, trial, hospice, nursing, whelping, deceased, transferred
    • +
+
+
+{/if}