// From /hooks/mutations/use-upload-document-sync.ts

// Generate a storage path using the folder UUID as the primary organization method
const timestamp = Date.now()

// Use 'vault-docs' as the root, followed by folder UUID (or 'unfiled' if none)
// This groups all files in the same folder together in storage
// Using folder UUIDs doesn't expose user IDs while maintaining organization
const folderPath = folderId || 'unfiled'

// Ensure filename uniqueness with timestamp prefix
const fileName = `${timestamp}_${name.replace(/[^a-zA-Z0-9]/g, '_')}`

// Final path: vault-docs/[folder-uuid]/[timestamp_filename]
const storagePath = `vault-docs/${folderPath}/${fileName}`

// Upload the file to storage
const { error: uploadError } = await supabase.storage
  .from('vault')
  .upload(storagePath, file, {
    cacheControl: '3600',
    upsert: false
  })
      folderId,
      name: values.name,
      file: selectedFile,
    })
    
    // Reset form and close dialog
    form.reset()
    setSelectedFile(null)
    onOpenChange(false) // Close the dialog
    
    // Refresh the document list after the document is uploaded
    setTimeout(() => {
      onDocumentUploaded() // Refresh the document list
    }, 500) // Small delay to ensure database consistency
    
  } catch (error) {
    console.error('Error uploading document:', error)
  }
}

// Lines 77-90 in /hooks/mutations/use-upload-document.ts
// Trigger document processing for embeddings
try {
  console.log(`Initiating document processing for document ID: ${data.id}`)
  
  const response = await fetch('/api/vault/process-document', {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
    },
    body: JSON.stringify({ documentId: data.id }),
  })

  // Error handling and status updates...
}

// Lines 68-75 in /app/api/vault/process-document/route.ts
// Initiate document processing in the background
void (async () => {
  try {
    // Process the document (extract text, chunk, embed, and store)
    const success = await processDocument(documentId, userId)
    
    // Status updates and error handling...
  } catch (error) {
    // Error handling...
  }
})()

/**
 * Fallback text search when vector search fails or returns no results
 * 
 * @param query - The search query text
 * @param clerkId - The user's Clerk ID for authentication
 * @param limit - Maximum number of results to return
 * @returns Promise containing an array of matching document chunks
 */
async function performTextSearch(query: string, clerkId: string, limit: number): Promise<any[]> {
  try {
    // Use text search (ilike) to find relevant documents
    const { data: textResults, error: textError } = await supabase
      .from('vault_document_chunks')
      .select(`
        id,
        document_id,
        content,
        metadata,
        vault_documents!inner(id, name, folder_uuid, vault_folders(uuid, name))
      `)
      .eq('clerk_id', clerkId)
      .ilike('content', `%${query}%`)
      .limit(limit)
    
    if (textError) {
      return []
    }
    
    return textResults || []
  } catch (error) {
    return []
  }
}

// From /lib/document-processor.ts
export async function extractTextFromDocument(
  documentId: string, 
  clerkId: string
): Promise<Document[] | null> {
  // Download file from storage
  // Determine file type
  // Extract text using appropriate method
  // Return LangChain Document objects with metadata
}

// From /lib/document-processor.ts
/**
 * Split documents into chunks for embedding using LangChain's RecursiveCharacterTextSplitter
 * 
 * @param docs - Array of Document objects to split into chunks
 * @returns Promise containing an array of chunked Document objects
 * 
 * This function uses RecursiveCharacterTextSplitter with a chunk size of 1000 characters
 * and 200 character overlap to create semantically meaningful chunks that preserve context
 * between chunks. The separators are configured to split on paragraphs, sentences, and words
 * in that order of preference.
 */
export async function splitDocumentsIntoChunks(docs: Document[]): Promise<Document[]> {
  // Create a text splitter with appropriate settings
  const splitter = new RecursiveCharacterTextSplitter({
    chunkSize: 1000,
    chunkOverlap: 200,
    separators: ["\n\n", "\n", ". ", " ", ""],
  })
  
  // Process each document and return chunks with enhanced metadata
}

// Lines 160-180 in /lib/document-processor.ts
export async function generateEmbeddings(
  text: string
): Promise<number[]> {
  const response = await openai.embeddings.create({
    model: "text-embedding-3-small",
    input: text,
    encoding_format: "float",
  })
  
  return response.data[0].embedding
}

// Lines 190-220 in /lib/document-processor.ts
export async function storeDocumentChunks(
  documentId: string,
  clerkId: string,
  chunks: Document[]
): Promise<boolean> {
  // Prepare chunks with metadata
  // Generate embeddings for each chunk
  // Store in vault_document_chunks table
}

// Lines 50-70 in /app/vault/page.tsx
const {
  documents,
  isLoading: isDocumentsLoading,
  refetch: refetchDocuments,
} = useVaultDocuments(selectedFolderId)

// Handler functions for user interactions
const handleDocumentUploaded = () => {
  // Force refetch documents to ensure the UI is updated
  refetchDocuments()
  
  // Set up a secondary refetch after a short delay to ensure database consistency
  setTimeout(() => {
    refetchDocuments()
  }, 1000)
  
  setIsUploadOpen(false)
}

// Folder selection and management
const handleFolderSelect = (folder: VaultFolder) => {
  onFolderSelect(folder.uuid)
  setSelectedFolder(folder)
}

// Lines 165-210 in /app/vault/components/document-list.tsx
// Helper function to render processing status
const renderProcessingStatus = (document: VaultDocument) => {
  // If the document is processed successfully, show a checkmark
  if (document.processed_for_embeddings === true || document.processing_status === 'completed') {
    return (
      <div className='flex items-center gap-1 px-2 py-0.5 text-xs rounded-full bg-green-100 text-green-800'>
        <CheckCircle2 className='h-3 w-3' />
        <span>Ready</span>
      </div>
    )
  }
  
  // Other status indicators...
}

// Lines 70-90 in /app/vault/components/upload-document.tsx
const onSubmit = async (values: FormValues) => {
  // Upload document and handle UI updates
}

// File type validation
const acceptedTypes = ['.pdf', '.doc', '.docx', '.txt', '.csv', '.html', '.htm']
const fileExtension = '.' + file.name.split('.').pop()?.toLowerCase()

if (!acceptedTypes.some(type => fileExtension === type)) {
  form.setError('file', { 
    type: 'manual', 
    message: 'Unsupported file type. Please upload PDF, Word, text, CSV, or HTML files.'
  })
  return
}

// Lines 72-115 in /app/vault/components/document-list.tsx
// Poll for document processing status
useEffect(() => {
  if (!documents?.length) return
  
  // Find documents that are pending or processing
  const docsToCheck = documents.filter(doc => 
    doc.processing_status === 'pending' || 
    doc.processing_status === 'processing' ||
    (doc.processed_for_embeddings === false && doc.processing_status !== 'failed')
  )
  
  // Add these documents to the processing set
  setProcessingDocuments(prev => {
    const newSet = new Set(prev)
    docsToCheck.forEach(doc => newSet.add(doc.id))
    return newSet
  })
  
  // If no documents need checking, return
  if (!docsToCheck.length) return
  
  // Set up polling for these documents
  const interval = setInterval(async () => {
    // Check status and update UI
  }, 3000) // Check every 3 seconds for faster updates
  
  return () => clearInterval(interval)
}, [documents, onDocumentDeleted])

// Lines 16-140 in /app/api/vault/process-document/route.ts
export async function POST(request: NextRequest) {
  try {
    // Authentication and validation
    // Update document status to 'processing'
    // Initiate background processing
    // Return quick response to client
  } catch (error) {
    // Error handling
  }
}

// Example usage in document-list.tsx
const {
  documents: processedDocs,
  completed: allCompleted,
  statusCounts
} = useDocumentProcessingStatus(
  documentIds,
  documentIds.length > 0
)

// Hook implementation in use-document-processing-status.ts
const { data, error } = await supabase
  .from('vault_documents')
  .select('id, processed_for_embeddings, processing_status')
  .in('id', documentIds)
  .eq('clerk_id', session.user.id)

// Check if all documents are processed
const allCompleted = data.every(doc => 
  doc.processed_for_embeddings === true || 
  doc.processing_status === 'completed' ||
  doc.processing_status === 'failed'
)

CREATE OR REPLACE FUNCTION match_documents(
  query_embedding VECTOR(1536),
  match_threshold FLOAT,
  match_count INT,
  p_clerk_id TEXT
)
RETURNS TABLE (
  id UUID,
  document_id TEXT,
  content TEXT,
  metadata JSONB,
  similarity FLOAT,
  document_name TEXT,
  folder_name TEXT
) 
LANGUAGE plpgsql
AS $$
BEGIN
  RETURN QUERY
  SELECT
    vdc.id,
    vdc.document_id::TEXT,
    vdc.content,
    vdc.metadata,
    (vdc.embedding <=> query_embedding) AS similarity,
    vd.name AS document_name,
    vf.name AS folder_name
  FROM
    vault_document_chunks vdc
  JOIN
    vault_documents vd ON vdc.document_id = vd.id
  JOIN
    vault_folders vf ON vd.folder_uuid = vf.uuid
  WHERE
    vdc.clerk_id = p_clerk_id
    AND (vdc.embedding <=> query_embedding) < match_threshold
  ORDER BY
    similarity ASC
  LIMIT
    match_count;
END;
$$;

// From /lib/document-processor.ts
/**
 * Search for similar document chunks based on a query
 * 
 * @param query - The search query text
 * @param clerkId - The user's Clerk ID for authentication
 * @param limit - Maximum number of results to return (default: 5)
 * @returns Promise containing an array of matching document chunks with metadata
 * 
 * This function performs a semantic search using vector embeddings to find
 * the most relevant document chunks based on the query. It first generates
 * an embedding for the query, then uses pgvector's similarity search to find
 * matching documents in the database.
 */
export async function searchDocuments(
  query: string,
  clerkId: string,
  limit: number = 5
): Promise<any[]> {
  try {
    // Generate embedding for the query using OpenAI API
    const embeddingResponse = await openai.embeddings.create({
      model: 'text-embedding-3-small',
      input: query,
      dimensions: 1536,
    })
    
    const queryEmbedding = embeddingResponse.data[0].embedding
    
    // Perform vector similarity search using pgvector
    const { data, error } = await supabase.rpc('match_documents', {
      query_embedding: queryEmbedding,
      match_threshold: 0.5, // Minimum similarity threshold
      match_count: limit,   // Maximum number of matches to return
      p_clerk_id: clerkId   // User's clerk ID for filtering
    })
    
    if (error || !data || data.length === 0) {
      // Fall back to text search if vector search fails
      return await performTextSearch(query, clerkId, limit)
    }
    
    // Format results with enhanced metadata and sort by similarity
    // ...
  } catch (error) {
    // Handle errors gracefully
    return []
  }
}

// Document search tool definition in chat system
searchVaultDocuments: tool({
  description: 'Search through the user\'s personal vault documents',
  execute: async ({ query, limit = 5 }) => {
    // Search for relevant document chunks
    const results = await searchDocuments(query, clerkId, limit)
    
    // Format results with proper citations and markdown formatting
    if (results.length === 0) {
      return "No relevant documents found."
    }
    
    return results.map(result => {
      return `**Document**: ${result.document_name} (Folder: ${result.folder_name})

${result.content}

---`
    }).join('\n')
  }
})

// Document citation guidelines in system prompt
DOCUMENT CITATIONS:
When you answer questions using information from the user's vault documents:
1. ALWAYS cite your sources by mentioning the document name in your response
2. Use the format "According to [Document Name]" or "Based on your [Document Name] in [Folder Name]"
3. If multiple documents are used, cite each one separately
4. If the document is in a specific folder, include that information too

Example citation formats:
- "According to your W-2 form, your income for 2024 was $75,000."
- "Based on your Tax Return (in Tax Documents folder), you claimed $12,000 in deductions."
- "Your Insurance Policy shows coverage of $500,000 for property damage."

FORMATTING GUIDELINES:
- Your responses will be rendered with full markdown support, so use markdown formatting to create scannable, well-organized content.
- Use **bold** for important numbers, percentages, and key insights.
- Use headings (## and ###) to organize longer responses into clear sections.
- Present financial data in tables when comparing multiple items or showing trends.
- Use bullet points for lists of related items.
- For code or calculations, use code blocks with ```.
- Keep paragraphs short (2-3 sentences) for better readability.

// Direct response style instructions
- DO NOT acknowledge what you're about to do (e.g., "I'll search through your documents" or "Let me check your data").
- SKIP all acknowledgment phrases and directly provide the answer or information.
- When using tools like document search, NEVER mention that you're searching - just present the information you find with proper citations.
- For financial data queries, directly present the data without saying "Let me gather that information".

// Example of user-based filtering in queries
.eq('clerk_id', userId)

// Example of server-side validation
if (!userId) {
  return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
}

if (!documentId) {
  return NextResponse.json(
    { error: 'Document ID is required' },
    { status: 400 }
  )
}

// Example of secure storage access
const { data: fileData, error: fileError } = await supabase
  .storage
  .from('vault')
  .download(document.path)

// Lines 55-75 in /app/api/vault/check-processing-status/route.ts
// For each document that's completed but doesn't have processed_for_embeddings set,
// update it to ensure consistency
const documentsToUpdate = data.filter(doc => 
  doc.processing_status === 'completed' && doc.processed_for_embeddings !== true
)

if (documentsToUpdate.length > 0) {
  console.log(`Fixing inconsistent status for ${documentsToUpdate.length} documents`)
  
  // Update each document to ensure processed_for_embeddings is set to true
  await Promise.all(documentsToUpdate.map(async (doc) => {
    // Update document status
  }))
}

// Lines 52-90 in /lib/document-processor.ts
// First try with the exact path
let fileData;
let fileError;

try {
  const result = await supabase.storage.from('vault').download(document.path)
  fileData = result.data
  fileError = result.error
  
  // If there's an error, try some variations of the path
  if (fileError || !fileData) {
    console.error('[DOCUMENT PROCESSOR] Error downloading with exact path:', fileError)
    console.log('[DOCUMENT PROCESSOR] Trying alternative paths...')
    
    // Try alternative paths
  }
}

// For DOCX files - First try DocxLoader, then fall back to mammoth
try {
  const loader = new DocxLoader(new Blob([fileData]))
  const documents = await loader.load()
  // Process documents...
} catch (docxError) {
  // Fallback to mammoth if DocxLoader fails
  const text = await extractTextFromDOCX(fileData)
  // Process text...
}

// For PDF files - Similar fallback pattern with pdf-parse