From 79a28d1f6072291fcebedcd1052da029c25ee494 Mon Sep 17 00:00:00 2001
From: starmorph <dylan@starmorph.com>
Date: Mon, 19 May 2025 11:14:28 -0700
Subject: [PATCH] fix file upload schema to match python langchain
 multimodality docs

---
 src/components/thread/index.tsx | 41 +++++++++++----------------------
 src/lib/multimodal-utils.ts     |  4 +---
 2 files changed, 15 insertions(+), 30 deletions(-)

diff --git a/src/components/thread/index.tsx b/src/components/thread/index.tsx
index 5d7a0ae..79198cb 100644
--- a/src/components/thread/index.tsx
+++ b/src/components/thread/index.tsx
@@ -181,18 +181,17 @@ export function Thread() {
     e.preventDefault();
     if (!input.trim() || isLoading) return;
     setFirstTokenReceived(false);
-
-    // TODO: check configurable object for modelname camelcase or snakecase else do openai format
-    const isOpenAI = true;
+    
+    // TODO: check configurable object for modelname camelcase or snakecase else do openai format 
+    const isOpenAI = true 
 
     const pdfBlocks = pdfUrlList.map(toOpenAIPDFBlock);
-
+ 
     const newHumanMessage: Message = {
       id: uuidv4(),
       type: "human",
       content: [
         { type: "text", text: input },
-        ...imageUrlList.map(toOpenAIImageBlock),
         ...pdfBlocks,
       ] as Message["content"],
     };
@@ -224,7 +223,7 @@ export function Thread() {
     const files = e.target.files;
     if (files) {
       const imageBlocks = await Promise.all(
-        Array.from(files).map(fileToImageBlock),
+        Array.from(files).map(fileToImageBlock)
       );
       setImageUrlList((prev) => [...prev, ...imageBlocks]);
     }
@@ -235,7 +234,7 @@ export function Thread() {
     const files = e.target.files;
     if (files) {
       const pdfBlocks = await Promise.all(
-        Array.from(files).map(fileToPDFBlock),
+        Array.from(files).map(fileToPDFBlock)
       );
       setPdfUrlList((prev) => [...prev, ...pdfBlocks]);
     }
@@ -277,26 +276,25 @@ export function Thread() {
       const imageFiles = files.filter((file) => file.type.startsWith("image/"));
       const pdfFiles = files.filter((file) => file.type === "application/pdf");
       const invalidFiles = files.filter(
-        (file) =>
-          !file.type.startsWith("image/") && file.type !== "application/pdf",
+        (file) => !file.type.startsWith("image/") && file.type !== "application/pdf"
       );
 
       if (invalidFiles.length > 0) {
         toast.error(
-          "You have uploaded invalid file type. Please upload an image or a PDF.",
+          "You have uploaded invalid file type. Please upload an image or a PDF."
         );
       }
 
       if (imageFiles.length) {
         const imageBlocks: Base64ContentBlock[] = await Promise.all(
-          imageFiles.map(fileToImageBlock),
+          imageFiles.map(fileToImageBlock)
         );
         setImageUrlList((prev) => [...prev, ...imageBlocks]);
       }
 
       if (pdfFiles.length) {
         const pdfBlocks: Base64ContentBlock[] = await Promise.all(
-          pdfFiles.map(fileToPDFBlock),
+          pdfFiles.map(fileToPDFBlock)
         );
         setPdfUrlList((prev) => [...prev, ...pdfBlocks]);
       }
@@ -522,10 +520,7 @@ export function Thread() {
                         {imageUrlList.map((imageBlock, idx) => {
                           const imageUrlString = `data:${imageBlock.mime_type};base64,${imageBlock.data}`;
                           return (
-                            <div
-                              className="relative"
-                              key={idx}
-                            >
+                            <div className="relative" key={idx}>
                               <img
                                 src={imageUrlString}
                                 alt="uploaded"
@@ -534,9 +529,7 @@ export function Thread() {
                               <CircleX
                                 className="absolute top-[2px] right-[2px] size-4 cursor-pointer rounded-full bg-gray-500 text-white"
                                 onClick={() =>
-                                  setImageUrlList(
-                                    imageUrlList.filter((_, i) => i !== idx),
-                                  )
+                                  setImageUrlList(imageUrlList.filter((_, i) => i !== idx))
                                 }
                               />
                             </div>
@@ -552,18 +545,12 @@ export function Thread() {
                             key={idx}
                           >
                             <span className="max-w-xs truncate text-sm">
-                              {String(
-                                pdfBlock.metadata?.filename ??
-                                  pdfBlock.metadata?.name ??
-                                  "",
-                              )}
+                              {String(pdfBlock.metadata?.filename ?? pdfBlock.metadata?.name ?? "")}
                             </span>
                             <CircleX
                               className="size-4 cursor-pointer text-teal-600 hover:text-teal-500"
                               onClick={() =>
-                                setPdfUrlList(
-                                  pdfUrlList.filter((_, i) => i !== idx),
-                                )
+                                setPdfUrlList(pdfUrlList.filter((_, i) => i !== idx))
                               }
                             />
                           </div>
diff --git a/src/lib/multimodal-utils.ts b/src/lib/multimodal-utils.ts
index 233bc41..b6360a3 100644
--- a/src/lib/multimodal-utils.ts
+++ b/src/lib/multimodal-utils.ts
@@ -31,13 +31,11 @@ export async function fileToPDFBlock(file: File): Promise<Base64ContentBlock> {
 // in lib/multimodal-utils.ts
 export function toOpenAIPDFBlock(block: Base64ContentBlock) {
   return {
-    type: "file",
-    file: {
+       type: "file",
       source_type: "base64",
       data: block.data,
       mime_type: block.mime_type ?? "application/pdf",
       filename: block.metadata?.name ?? block.metadata?.filename ?? "file.pdf",
-    },
   };
 }