kexsh/agent-chat-ui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix file upload schema to match python langchain multimodality docs

Browse Source
This commit is contained in:
starmorph
2025-05-19 11:14:28 -07:00
parent fcaa963085
commit 79a28d1f60
2 changed files with 15 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
src/components/thread/index.tsx
View File

@@ -181,18 +181,17 @@ export function Thread() {
e.preventDefault(); e.preventDefault();
if (!input.trim() || isLoading) return; if (!input.trim() || isLoading) return;
setFirstTokenReceived(false); setFirstTokenReceived(false);
// TODO: check configurable object for modelname camelcase or snakecase else do openai format // TODO: check configurable object for modelname camelcase or snakecase else do openai format
const isOpenAI = true; const isOpenAI = true
const pdfBlocks = pdfUrlList.map(toOpenAIPDFBlock); const pdfBlocks = pdfUrlList.map(toOpenAIPDFBlock);
const newHumanMessage: Message = { const newHumanMessage: Message = {
id: uuidv4(), id: uuidv4(),
type: "human", type: "human",
content: [ content: [
{ type: "text", text: input }, { type: "text", text: input },
...imageUrlList.map(toOpenAIImageBlock),
...pdfBlocks, ...pdfBlocks,
] as Message["content"], ] as Message["content"],
}; };
@@ -224,7 +223,7 @@ export function Thread() {
const files = e.target.files; const files = e.target.files;
if (files) { if (files) {
const imageBlocks = await Promise.all( const imageBlocks = await Promise.all(
Array.from(files).map(fileToImageBlock), Array.from(files).map(fileToImageBlock)
); );
setImageUrlList((prev) => [...prev, ...imageBlocks]); setImageUrlList((prev) => [...prev, ...imageBlocks]);
} }
@@ -235,7 +234,7 @@ export function Thread() {
const files = e.target.files; const files = e.target.files;
if (files) { if (files) {
const pdfBlocks = await Promise.all( const pdfBlocks = await Promise.all(
Array.from(files).map(fileToPDFBlock), Array.from(files).map(fileToPDFBlock)
); );
setPdfUrlList((prev) => [...prev, ...pdfBlocks]); setPdfUrlList((prev) => [...prev, ...pdfBlocks]);
} }
@@ -277,26 +276,25 @@ export function Thread() {
const imageFiles = files.filter((file) => file.type.startsWith("image/")); const imageFiles = files.filter((file) => file.type.startsWith("image/"));
const pdfFiles = files.filter((file) => file.type === "application/pdf"); const pdfFiles = files.filter((file) => file.type === "application/pdf");
const invalidFiles = files.filter( const invalidFiles = files.filter(
(file) => (file) => !file.type.startsWith("image/") && file.type !== "application/pdf"
!file.type.startsWith("image/") && file.type !== "application/pdf",
); );
if (invalidFiles.length > 0) { if (invalidFiles.length > 0) {
toast.error( toast.error(
"You have uploaded invalid file type. Please upload an image or a PDF.", "You have uploaded invalid file type. Please upload an image or a PDF."
); );
} }
if (imageFiles.length) { if (imageFiles.length) {
const imageBlocks: Base64ContentBlock[] = await Promise.all( const imageBlocks: Base64ContentBlock[] = await Promise.all(
imageFiles.map(fileToImageBlock), imageFiles.map(fileToImageBlock)
); );
setImageUrlList((prev) => [...prev, ...imageBlocks]); setImageUrlList((prev) => [...prev, ...imageBlocks]);
} }
if (pdfFiles.length) { if (pdfFiles.length) {
const pdfBlocks: Base64ContentBlock[] = await Promise.all( const pdfBlocks: Base64ContentBlock[] = await Promise.all(
pdfFiles.map(fileToPDFBlock), pdfFiles.map(fileToPDFBlock)
); );
setPdfUrlList((prev) => [...prev, ...pdfBlocks]); setPdfUrlList((prev) => [...prev, ...pdfBlocks]);
} }
@@ -522,10 +520,7 @@ export function Thread() {
{imageUrlList.map((imageBlock, idx) => { {imageUrlList.map((imageBlock, idx) => {
const imageUrlString = `data:${imageBlock.mime_type};base64,${imageBlock.data}`; const imageUrlString = `data:${imageBlock.mime_type};base64,${imageBlock.data}`;
return ( return (
<div <div className="relative" key={idx}>
className="relative"
key={idx}
>
<img <img
src={imageUrlString} src={imageUrlString}
alt="uploaded" alt="uploaded"
@@ -534,9 +529,7 @@ export function Thread() {
<CircleX <CircleX
className="absolute top-[2px] right-[2px] size-4 cursor-pointer rounded-full bg-gray-500 text-white" className="absolute top-[2px] right-[2px] size-4 cursor-pointer rounded-full bg-gray-500 text-white"
onClick={() => onClick={() =>
setImageUrlList( setImageUrlList(imageUrlList.filter((_, i) => i !== idx))
imageUrlList.filter((_, i) => i !== idx),
)
} }
/> />
</div> </div>
@@ -552,18 +545,12 @@ export function Thread() {
key={idx} key={idx}
> >
<span className="max-w-xs truncate text-sm"> <span className="max-w-xs truncate text-sm">
{String( {String(pdfBlock.metadata?.filename ?? pdfBlock.metadata?.name ?? "")}
pdfBlock.metadata?.filename ??
pdfBlock.metadata?.name ??
"",
)}
</span> </span>
<CircleX <CircleX
className="size-4 cursor-pointer text-teal-600 hover:text-teal-500" className="size-4 cursor-pointer text-teal-600 hover:text-teal-500"
onClick={() => onClick={() =>
setPdfUrlList( setPdfUrlList(pdfUrlList.filter((_, i) => i !== idx))
pdfUrlList.filter((_, i) => i !== idx),
)
} }
/> />
</div> </div>

4
src/lib/multimodal-utils.ts
View File

@@ -31,13 +31,11 @@ export async function fileToPDFBlock(file: File): Promise<Base64ContentBlock> {
// in lib/multimodal-utils.ts // in lib/multimodal-utils.ts
export function toOpenAIPDFBlock(block: Base64ContentBlock) { export function toOpenAIPDFBlock(block: Base64ContentBlock) {
return { return {
type: "file", type: "file",
file: {
source_type: "base64", source_type: "base64",
data: block.data, data: block.data,
mime_type: block.mime_type ?? "application/pdf", mime_type: block.mime_type ?? "application/pdf",
filename: block.metadata?.name ?? block.metadata?.filename ?? "file.pdf", filename: block.metadata?.name ?? block.metadata?.filename ?? "file.pdf",
},
}; };
} }