mirror of
https://github.com/tiennm99/DocsGPT.git
synced 2026-10-05 06:13:15 +00:00
Render chunk token counts through one tested helper
Both chunk headers inlined the same `token_count ? … : '-'` expression, and `ChunkType` declared every metadata value a string even though the JSON-typed stores return the count as a number. Move the formatting into `formatChunkTokens`, which accepts either shape and only falls back to a dash when the value is genuinely unusable, and type `ChunkType.metadata` to match what the API actually returns.
This commit is contained in:
1 parent
d787f54c2d
commit
3954de9b9e
4 files changed
+72
-7
No files matched your search
@@ -20,6 +20,7 @@ import ConfirmationModal from '../modals/ConfirmationModal';
|
||||
import { ActiveState } from '../models/misc';
|
||||
import { selectToken } from '../preferences/preferenceSlice';
|
||||
import { ChunkType } from '../settings/types';
|
||||
import { formatChunkTokens } from './chunkUtils';
|
||||
import Pagination from './DocumentPagination';
|
||||
import SkeletonLoader from './SkeletonLoader';
|
||||
import { Button } from './ui/button';
|
||||
@@ -628,9 +629,7 @@ const Chunks: React.FC<ChunksProps> = ({
|
||||
<div className="w-full">
|
||||
<div className="border-border bg-muted dark:border-border dark:bg-card flex w-full items-center justify-between border-b px-4 py-3">
|
||||
<div className="dark:text-muted-foreground text-sm text-[#59636E]">
|
||||
{chunk.metadata.token_count
|
||||
? chunk.metadata.token_count.toLocaleString()
|
||||
: '-'}{' '}
|
||||
{formatChunkTokens(chunk.metadata)}{' '}
|
||||
{t('settings.sources.tokensUnit')}
|
||||
</div>
|
||||
</div>
|
||||
@@ -663,9 +662,7 @@ const Chunks: React.FC<ChunksProps> = ({
|
||||
<div className="border-border dark:border-border relative flex w-full flex-col overflow-hidden rounded-md border">
|
||||
<div className="border-border bg-muted dark:border-border dark:bg-card flex w-full items-center justify-between border-b px-4 py-3">
|
||||
<div className="dark:text-muted-foreground text-sm text-[#59636E]">
|
||||
{editingChunk.metadata.token_count
|
||||
? editingChunk.metadata.token_count.toLocaleString()
|
||||
: '-'}{' '}
|
||||
{formatChunkTokens(editingChunk.metadata)}{' '}
|
||||
{t('settings.sources.tokensUnit')}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { formatChunkTokens, UNKNOWN_TOKEN_COUNT } from './chunkUtils';
|
||||
|
||||
type ChunkMetadata = Parameters<typeof formatChunkTokens>[0];
|
||||
|
||||
describe('formatChunkTokens', () => {
|
||||
it('formats a numeric count with separators', () => {
|
||||
expect(formatChunkTokens({ token_count: 1234 })).toBe(
|
||||
(1234).toLocaleString(),
|
||||
);
|
||||
});
|
||||
|
||||
it('formats a count a store handed back as a string', () => {
|
||||
expect(formatChunkTokens({ token_count: '1234' })).toBe(
|
||||
(1234).toLocaleString(),
|
||||
);
|
||||
});
|
||||
|
||||
it('falls back to a dash when the count is unusable', () => {
|
||||
for (const token_count of [undefined, 0, -1, 'abc', '']) {
|
||||
expect(formatChunkTokens({ token_count })).toBe(UNKNOWN_TOKEN_COUNT);
|
||||
}
|
||||
});
|
||||
|
||||
it('tolerates metadata the store returned as null', () => {
|
||||
expect(formatChunkTokens(null as unknown as ChunkMetadata)).toBe(
|
||||
UNKNOWN_TOKEN_COUNT,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,27 @@
|
||||
/**
|
||||
* Pure helpers for the chunk cards in the source viewer. Kept free of React so
|
||||
* the formatting rules are unit-testable in isolation.
|
||||
*/
|
||||
|
||||
import { ChunkType } from '../settings/types';
|
||||
|
||||
/** What a chunk card shows when the token count is genuinely unknown. */
|
||||
export const UNKNOWN_TOKEN_COUNT = '-';
|
||||
|
||||
/**
|
||||
* Render a chunk's token count for display.
|
||||
*
|
||||
* The backend fills the count in for chunks that were indexed without one, but
|
||||
* stores round-trip metadata types differently: pgvector keeps JSON numbers
|
||||
* while other backends hand the same value back as a string. Both are counts,
|
||||
* so both get thousands separators; only a missing or unusable value falls
|
||||
* back to a dash.
|
||||
*/
|
||||
export function formatChunkTokens(metadata: ChunkType['metadata']): string {
|
||||
const raw = metadata?.token_count;
|
||||
const count = typeof raw === 'string' ? Number(raw.trim()) : raw;
|
||||
if (typeof count !== 'number' || !Number.isFinite(count) || count <= 0) {
|
||||
return UNKNOWN_TOKEN_COUNT;
|
||||
}
|
||||
return count.toLocaleString();
|
||||
}
|
||||
@@ -3,7 +3,17 @@ import { ConfigRequirements } from '../../modals/types';
|
||||
export type ChunkType = {
|
||||
doc_id: string;
|
||||
text: string;
|
||||
metadata: { [key: string]: string };
|
||||
/**
|
||||
* Chunk metadata as the vector store recorded it. Values are strings for
|
||||
* most backends, but numbers (`token_count`) survive the round trip on the
|
||||
* JSON-typed ones, so consumers must handle both.
|
||||
*/
|
||||
metadata: {
|
||||
title?: string;
|
||||
source?: string;
|
||||
token_count?: number | string;
|
||||
[key: string]: number | string | undefined;
|
||||
};
|
||||
};
|
||||
|
||||
export type LogEventType =
|
||||
|
||||
Reference in new issue
Block a user