From 3954de9b9ea6954bf4fa9bb5aab48ffbfd922c3d Mon Sep 17 00:00:00 2001 From: arc53-machine <232052973+arc53-machine@users.noreply.github.com> Date: Tue, 22 Sep 2026 13:45:40 +0100 Subject: [PATCH] Render chunk token counts through one tested helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both chunk headers inlined the same `token_count ? … : '-'` expression, and `ChunkType` declared every metadata value a string even though the JSON-typed stores return the count as a number. Move the formatting into `formatChunkTokens`, which accepts either shape and only falls back to a dash when the value is genuinely unusable, and type `ChunkType.metadata` to match what the API actually returns. --- frontend/src/components/Chunks.tsx | 9 +++---- frontend/src/components/chunkUtils.test.ts | 31 ++++++++++++++++++++++ frontend/src/components/chunkUtils.ts | 27 +++++++++++++++++++ frontend/src/settings/types/index.ts | 12 ++++++++- 4 files changed, 72 insertions(+), 7 deletions(-) create mode 100644 frontend/src/components/chunkUtils.test.ts create mode 100644 frontend/src/components/chunkUtils.ts diff --git a/frontend/src/components/Chunks.tsx b/frontend/src/components/Chunks.tsx index e01ead13..8efa7f64 100644 --- a/frontend/src/components/Chunks.tsx +++ b/frontend/src/components/Chunks.tsx @@ -20,6 +20,7 @@ import ConfirmationModal from '../modals/ConfirmationModal'; import { ActiveState } from '../models/misc'; import { selectToken } from '../preferences/preferenceSlice'; import { ChunkType } from '../settings/types'; +import { formatChunkTokens } from './chunkUtils'; import Pagination from './DocumentPagination'; import SkeletonLoader from './SkeletonLoader'; import { Button } from './ui/button'; @@ -628,9 +629,7 @@ const Chunks: React.FC = ({
- {chunk.metadata.token_count - ? chunk.metadata.token_count.toLocaleString() - : '-'}{' '} + {formatChunkTokens(chunk.metadata)}{' '} {t('settings.sources.tokensUnit')}
@@ -663,9 +662,7 @@ const Chunks: React.FC = ({
- {editingChunk.metadata.token_count - ? editingChunk.metadata.token_count.toLocaleString() - : '-'}{' '} + {formatChunkTokens(editingChunk.metadata)}{' '} {t('settings.sources.tokensUnit')}
diff --git a/frontend/src/components/chunkUtils.test.ts b/frontend/src/components/chunkUtils.test.ts new file mode 100644 index 00000000..6ff36b9a --- /dev/null +++ b/frontend/src/components/chunkUtils.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from 'vitest'; + +import { formatChunkTokens, UNKNOWN_TOKEN_COUNT } from './chunkUtils'; + +type ChunkMetadata = Parameters[0]; + +describe('formatChunkTokens', () => { + it('formats a numeric count with separators', () => { + expect(formatChunkTokens({ token_count: 1234 })).toBe( + (1234).toLocaleString(), + ); + }); + + it('formats a count a store handed back as a string', () => { + expect(formatChunkTokens({ token_count: '1234' })).toBe( + (1234).toLocaleString(), + ); + }); + + it('falls back to a dash when the count is unusable', () => { + for (const token_count of [undefined, 0, -1, 'abc', '']) { + expect(formatChunkTokens({ token_count })).toBe(UNKNOWN_TOKEN_COUNT); + } + }); + + it('tolerates metadata the store returned as null', () => { + expect(formatChunkTokens(null as unknown as ChunkMetadata)).toBe( + UNKNOWN_TOKEN_COUNT, + ); + }); +}); diff --git a/frontend/src/components/chunkUtils.ts b/frontend/src/components/chunkUtils.ts new file mode 100644 index 00000000..56853106 --- /dev/null +++ b/frontend/src/components/chunkUtils.ts @@ -0,0 +1,27 @@ +/** + * Pure helpers for the chunk cards in the source viewer. Kept free of React so + * the formatting rules are unit-testable in isolation. + */ + +import { ChunkType } from '../settings/types'; + +/** What a chunk card shows when the token count is genuinely unknown. */ +export const UNKNOWN_TOKEN_COUNT = '-'; + +/** + * Render a chunk's token count for display. + * + * The backend fills the count in for chunks that were indexed without one, but + * stores round-trip metadata types differently: pgvector keeps JSON numbers + * while other backends hand the same value back as a string. Both are counts, + * so both get thousands separators; only a missing or unusable value falls + * back to a dash. + */ +export function formatChunkTokens(metadata: ChunkType['metadata']): string { + const raw = metadata?.token_count; + const count = typeof raw === 'string' ? Number(raw.trim()) : raw; + if (typeof count !== 'number' || !Number.isFinite(count) || count <= 0) { + return UNKNOWN_TOKEN_COUNT; + } + return count.toLocaleString(); +} diff --git a/frontend/src/settings/types/index.ts b/frontend/src/settings/types/index.ts index fe662e7b..81e557a8 100644 --- a/frontend/src/settings/types/index.ts +++ b/frontend/src/settings/types/index.ts @@ -3,7 +3,17 @@ import { ConfigRequirements } from '../../modals/types'; export type ChunkType = { doc_id: string; text: string; - metadata: { [key: string]: string }; + /** + * Chunk metadata as the vector store recorded it. Values are strings for + * most backends, but numbers (`token_count`) survive the round trip on the + * JSON-typed ones, so consumers must handle both. + */ + metadata: { + title?: string; + source?: string; + token_count?: number | string; + [key: string]: number | string | undefined; + }; }; export type LogEventType =