Render chunk token counts through one tested helper

Both chunk headers inlined the same `token_count ? … : '-'` expression, and
`ChunkType` declared every metadata value a string even though the JSON-typed
stores return the count as a number.

Move the formatting into `formatChunkTokens`, which accepts either shape and
only falls back to a dash when the value is genuinely unusable, and type
`ChunkType.metadata` to match what the API actually returns.
This commit is contained in:
arc53-machine committed 2026-09-22 13:45:40 +01:00
1 parent d787f54c2d
commit 3954de9b9e
4 files changed
+72 -7

No files matched your search

+3 -6
View File
@@ -20,6 +20,7 @@ import ConfirmationModal from '../modals/ConfirmationModal';
import { ActiveState } from '../models/misc';
import { selectToken } from '../preferences/preferenceSlice';
import { ChunkType } from '../settings/types';
import { formatChunkTokens } from './chunkUtils';
import Pagination from './DocumentPagination';
import SkeletonLoader from './SkeletonLoader';
import { Button } from './ui/button';
@@ -628,9 +629,7 @@ const Chunks: React.FC<ChunksProps> = ({
<div className="w-full">
<div className="border-border bg-muted dark:border-border dark:bg-card flex w-full items-center justify-between border-b px-4 py-3">
<div className="dark:text-muted-foreground text-sm text-[#59636E]">
{chunk.metadata.token_count
? chunk.metadata.token_count.toLocaleString()
: '-'}{' '}
{formatChunkTokens(chunk.metadata)}{' '}
{t('settings.sources.tokensUnit')}
</div>
</div>
@@ -663,9 +662,7 @@ const Chunks: React.FC<ChunksProps> = ({
<div className="border-border dark:border-border relative flex w-full flex-col overflow-hidden rounded-md border">
<div className="border-border bg-muted dark:border-border dark:bg-card flex w-full items-center justify-between border-b px-4 py-3">
<div className="dark:text-muted-foreground text-sm text-[#59636E]">
{editingChunk.metadata.token_count
? editingChunk.metadata.token_count.toLocaleString()
: '-'}{' '}
{formatChunkTokens(editingChunk.metadata)}{' '}
{t('settings.sources.tokensUnit')}
</div>
</div>
@@ -0,0 +1,31 @@
import { describe, expect, it } from 'vitest';
import { formatChunkTokens, UNKNOWN_TOKEN_COUNT } from './chunkUtils';
type ChunkMetadata = Parameters<typeof formatChunkTokens>[0];
describe('formatChunkTokens', () => {
it('formats a numeric count with separators', () => {
expect(formatChunkTokens({ token_count: 1234 })).toBe(
(1234).toLocaleString(),
);
});
it('formats a count a store handed back as a string', () => {
expect(formatChunkTokens({ token_count: '1234' })).toBe(
(1234).toLocaleString(),
);
});
it('falls back to a dash when the count is unusable', () => {
for (const token_count of [undefined, 0, -1, 'abc', '']) {
expect(formatChunkTokens({ token_count })).toBe(UNKNOWN_TOKEN_COUNT);
}
});
it('tolerates metadata the store returned as null', () => {
expect(formatChunkTokens(null as unknown as ChunkMetadata)).toBe(
UNKNOWN_TOKEN_COUNT,
);
});
});
+27
View File
@@ -0,0 +1,27 @@
/**
* Pure helpers for the chunk cards in the source viewer. Kept free of React so
* the formatting rules are unit-testable in isolation.
*/
import { ChunkType } from '../settings/types';
/** What a chunk card shows when the token count is genuinely unknown. */
export const UNKNOWN_TOKEN_COUNT = '-';
/**
* Render a chunk's token count for display.
*
* The backend fills the count in for chunks that were indexed without one, but
* stores round-trip metadata types differently: pgvector keeps JSON numbers
* while other backends hand the same value back as a string. Both are counts,
* so both get thousands separators; only a missing or unusable value falls
* back to a dash.
*/
export function formatChunkTokens(metadata: ChunkType['metadata']): string {
const raw = metadata?.token_count;
const count = typeof raw === 'string' ? Number(raw.trim()) : raw;
if (typeof count !== 'number' || !Number.isFinite(count) || count <= 0) {
return UNKNOWN_TOKEN_COUNT;
}
return count.toLocaleString();
}
+11 -1
View File
@@ -3,7 +3,17 @@ import { ConfigRequirements } from '../../modals/types';
export type ChunkType = {
doc_id: string;
text: string;
metadata: { [key: string]: string };
/**
* Chunk metadata as the vector store recorded it. Values are strings for
* most backends, but numbers (`token_count`) survive the round trip on the
* JSON-typed ones, so consumers must handle both.
*/
metadata: {
title?: string;
source?: string;
token_count?: number | string;
[key: string]: number | string | undefined;
};
};
export type LogEventType =