Files
backend/test/tbank-read-baseline.e2e-spec.ts
T

608 lines
22 KiB
TypeScript

import type { NextFunction, Request, Response } from 'express';
import type { Server } from 'node:http';
import { INestApplication } from '@nestjs/common';
import { JwtService } from '@nestjs/jwt';
import { Test } from '@nestjs/testing';
import assert from 'node:assert/strict';
import { mkdtempSync, readFileSync, statSync, writeFileSync } from 'node:fs';
import { cpus, tmpdir, totalmem } from 'node:os';
import { join } from 'node:path';
import { performance } from 'node:perf_hooks';
import { setTimeout as delay } from 'node:timers/promises';
import request from 'supertest';
import { AppModule } from '../src/app.module';
import { MoexClient } from '../src/integrations/moex/moex.api';
import { NcrRatingsClient } from '../src/integrations/ncr/ncr.api';
import { TbankTokenService } from '../src/integrations/tbank/tbank-token.service';
import {
TbankClient,
TbankOperationsByCursorRequest,
} from '../src/integrations/tbank/tbank.client';
import { configureApp } from '../src/main';
import { PrismaService } from '../src/prisma/prisma.service';
const evidenceDirectory = mkdtempSync(join(tmpdir(), 'wealthledger-read-evidence-'));
// Same navigation and synthetic sources as the saved baseline; isolated SQLite only.
describe('T-Bank persisted read budget', () => {
let app: INestApplication<Server>;
const owner = 'synthetic-baseline-owner';
const accounts = Array.from({ length: 3 }, (_, index) => ({
id: `baseline-${index}`,
name: `Synthetic ${index}`,
type: 'ACCOUNT_TYPE_TINKOFF',
openedDate: { seconds: Date.UTC(2016, 8, 1) / 1000 },
}));
const instruments = Array.from({ length: 200 }, (_, index) => ({
uid: `00000000-0000-4000-8000-${String(index).padStart(12, '0')}`,
ticker: `SYNTH${index}`,
name: `Synthetic instrument ${index}`,
assetUid: `asset-${index}`,
isin: `SYNTHETIC${index}`,
currency: 'rub',
instrumentKind:
index % 2 ? 'INSTRUMENT_TYPE_BOND' : 'INSTRUMENT_TYPE_SHARE',
classCode: index % 2 ? 'TQOB' : 'TQBR',
apiTradeAvailableFlag: true,
first_1dayCandleDate: { seconds: Date.UTC(2026, 7, 31 - 4999) / 1000 },
nominal: { currency: 'rub', units: '1000' },
aciValue: { currency: 'rub', units: '12', nano: 345678900 },
}));
const candles = Array.from({ length: 5000 }, (_, index) => ({
time: { seconds: Date.UTC(2026, 7, 31 - 4999 + index) / 1000 },
close: { units: '100', nano: index },
isComplete: true,
}));
const operations = Array.from({ length: 100_000 }, (_, index) => {
// 1,000 measured-month rows, 200 income receipts, remaining rows over 10 years.
const time =
index < 1000
? Date.UTC(2026, 7, 1) + index * 2_500_000
: Date.UTC(2016, 8, 1) + (index - 1000) * 3_160_000;
const instrument = instruments[index % instruments.length];
return {
id: `operation-${index}`,
date: { seconds: String(Math.floor(time / 1000)) },
type: index % 5 === 0 ? 'OPERATION_TYPE_COUPON' : 'OPERATION_TYPE_BUY',
instrumentUid: instrument.uid,
ticker: instrument.ticker,
instrumentType: 'bond',
payment: { currency: 'rub', units: '123', nano: 123456789 },
quantity: '1',
quantityDone: '1',
};
}).sort((a, b) => Number(b.date.seconds) - Number(a.date.seconds));
const rpc: Record<string, number> = {};
let upstreamMode: 'delayed' | 'failed' | 'ready' = 'ready';
let sourceGate: Promise<void> | undefined;
let releaseSource = () => {};
const serverTimes: number[] = [];
async function counted<T>(name: string, value: T): Promise<T> {
// eslint-disable-next-line security/detect-object-injection -- Keys come from local test fixtures or numeric iteration, not external input.
rpc[name] = (rpc[name] ?? 0) + 1;
if (upstreamMode === 'delayed') await sourceGate;
if (upstreamMode === 'failed') {
return Promise.reject(new Error('Synthetic outage'));
}
return Promise.resolve(value);
}
const upstream = {
getOpenAccounts: () => counted('accounts', accounts),
getPortfolio: () =>
counted('portfolio', {
positions: instruments.map((instrument, index) => ({
instrumentUid: instrument.uid,
instrumentType: index % 2 ? 'bond' : 'share',
quantity: { units: '1' },
currentPrice: { currency: 'rub', units: '1000' },
})),
}),
getPositions: () =>
counted('positions', { money: [{ currency: 'rub', units: '100' }] }),
getInstrumentByUid: (_token: string, uid: string) =>
counted(
'metadata',
instruments.find((instrument) => instrument.uid === uid),
),
findShares: (_token: string, query: string) =>
counted(
'findShares',
instruments.filter(
(item) =>
item.instrumentKind === 'INSTRUMENT_TYPE_SHARE' &&
item.ticker.includes(query),
),
),
findBonds: (_token: string, query: string) =>
counted(
'findBonds',
instruments.filter(
(item) =>
item.instrumentKind === 'INSTRUMENT_TYPE_BOND' &&
item.ticker.includes(query),
),
),
getShare: (_token: string, uid: string) =>
counted(
'share',
instruments.find((item) => item.uid === uid),
),
getBond: (_token: string, uid: string) =>
counted(
'bond',
instruments.find((item) => item.uid === uid),
),
getLastPrice: (_token: string, uid: string) =>
counted('lastPrice', {
instrumentUid: uid,
price: { units: '100' },
time: { seconds: Date.UTC(2026, 7, 31) / 1000 },
}),
getLastPrices: (_token: string, uids: string[]) =>
counted(
'lastPrices',
uids.map((uid) => ({ instrumentUid: uid, price: { units: '100' } })),
),
getTradingStatus: () =>
counted('tradingStatus', 'SECURITY_TRADING_STATUS_NORMAL_TRADING'),
getCandles: (_token: string, _uid: string, from: Date, to: Date) =>
counted(
'candles',
candles.filter(
(row) =>
row.time.seconds * 1000 >= from.valueOf() &&
row.time.seconds * 1000 <= to.valueOf(),
),
),
getAssetDescription: () => counted('assetDescription', 'Synthetic company'),
getFundamentals: () =>
counted('fundamentals', {
currency: 'rub',
revenueTtm: 1_000_000,
peRatioTtm: 5,
}),
getDividends: () =>
counted('dividends', [
{
dividendNet: { currency: 'rub', units: '10' },
recordDate: { seconds: Date.UTC(2026, 9, 1) / 1000 },
},
]),
getBondCoupons: () =>
counted('coupons', [
{
payOneBond: { currency: 'rub', units: '10' },
couponDate: { seconds: Date.UTC(2026, 9, 1) / 1000 },
fixDate: { seconds: Date.UTC(2026, 8, 30) / 1000 },
couponPeriod: 30,
},
]),
getOperationsByCursor: (
_token: string,
query: TbankOperationsByCursorRequest,
) => {
const rows = operations.filter(
(row) =>
Number(row.date.seconds) >= Number(query.from?.seconds ?? 0) &&
Number(row.date.seconds) <= Number(query.to?.seconds ?? Infinity),
);
const offset = Number(query.cursor ?? 0);
const end = offset + (query.limit ?? 50);
return counted('operations', {
items: rows.slice(offset, end),
hasNext: end < rows.length,
nextCursor: end < rows.length ? String(end) : '',
});
},
};
const token = new JwtService().sign(
{ sub: owner, type: 'access' },
{ secret: process.env.JWT_SECRET },
);
async function start() {
const module = await Test.createTestingModule({ imports: [AppModule] })
.overrideProvider(TbankClient)
.useValue(upstream)
.overrideProvider(MoexClient)
.useValue({
bondEvents: () =>
counted('moexEvents', [
{
type: 'coupon',
date: '2026-10-01',
value: '10',
currency: 'RUB',
},
]),
bondYield: () => counted('moexYield', null),
})
.overrideProvider(NcrRatingsClient)
.useValue({ rating: () => counted('ncrRating', null) })
.compile();
app = module.createNestApplication();
app.use((_req: Request, res: Response, next: NextFunction) => {
const start = performance.now();
res.once('finish', () => serverTimes.push(performance.now() - start));
next();
});
configureApp(app);
await app.init();
await app.listen(0, '127.0.0.1');
}
beforeAll(async () => {
await start();
const prisma = app.get(PrismaService);
await prisma.user.create({
data: {
id: owner,
email: 'baseline@example.invalid',
passwordHash: 'unused',
},
});
const connection = await prisma.tbankConnection.create({
data: {
userId: owner,
encryptedToken: app.get(TbankTokenService).encrypt('synthetic-token'),
connectedAt: new Date(),
membershipAt: new Date(),
membershipRevision: 1,
},
});
const now = new Date();
for (const [index, account] of accounts.entries()) {
const scope = {
userId: owner,
revision: connection.revision,
accountId: account.id,
};
await prisma.tbankAccount.create({
data: {
...scope,
name: account.name,
type: 'BROKERAGE',
openedOn: '2016-09-01',
upstreamOrder: index,
historyFrom: new Date('2016-09-01Z'),
historyBoundary: 'openedOn',
},
});
for (let offset = 0; offset < operations.length; offset += 1000) {
await prisma.tbankOperation.createMany({
data: operations.slice(offset, offset + 1000).map((row) => ({
...scope,
brokerOperationId: row.id,
occurredAt: new Date(Number(row.date.seconds) * 1000),
instrumentUid: row.instrumentUid,
category: row.type === 'OPERATION_TYPE_COUPON' ? 'income' : 'trade',
payload: row,
})),
});
}
await prisma.tbankOperationCoverage.create({
data: {
...scope,
from: new Date('2016-08-31T21:00:00Z'),
to: now,
fetchedAt: now,
asOf: now,
},
});
}
for (const instrument of instruments) {
await prisma.tbankInstrument.create({
data: {
provider: 'tbank',
uid: instrument.uid,
ticker: instrument.ticker,
instrumentType:
instrument.instrumentKind === 'INSTRUMENT_TYPE_SHARE'
? 'share'
: 'bond',
metadata: instrument,
fetchedAt: now,
},
});
await prisma.$executeRawUnsafe(
`WITH RECURSIVE days(n) AS (SELECT 0 UNION ALL SELECT n+1 FROM days WHERE n<4999)
INSERT INTO TbankCandle(provider,instrumentUid,interval,time,close,payload,isComplete,fetchedAt)
SELECT 'tbank',?,'day',strftime('%Y-%m-%dT%H:%M:%fZ', (?+n*86400000)/1000.0, 'unixepoch'),'100','{}',1,? FROM days`,
instrument.uid,
candles[0].time.seconds * 1000,
now.toISOString(),
);
const today = new Date(now);
today.setUTCHours(0, 0, 0, 0);
await prisma.tbankCandleCoverage.createMany({
data: [
{
provider: 'tbank',
instrumentUid: instrument.uid,
interval: 'day',
from: new Date(candles[0].time.seconds * 1000),
to: today,
isComplete: true,
fetchedAt: now,
},
{
provider: 'tbank',
instrumentUid: instrument.uid,
interval: 'day',
from: today,
to: new Date(today.valueOf() + 86400000),
isComplete: false,
fetchedAt: now,
},
],
});
}
assert.equal(await prisma.tbankOperation.count(), 300_000);
assert.equal(await prisma.tbankCandle.count(), 1_000_000);
}, 300_000);
afterAll(async () => app?.close());
it('measures the same navigation with 10 clients and 100 reads per family', async () => {
expect(operations).toHaveLength(100_000);
expect(candles).toHaveLength(5000);
const routes = [
'/brokerage-accounts',
'/brokerage-accounts/baseline-0/portfolio',
'/brokerage-accounts/all/portfolio',
'/brokerage-accounts/baseline-0/operations?from=2026-08-01&to=2026-08-31&limit=50',
'/brokerage-accounts/all/operations?from=2026-08-01&to=2026-08-31&limit=50',
'/brokerage-accounts/baseline-0/income-analytics?from=2026-08-01&to=2026-08-31',
'/brokerage-accounts/all/income-analytics?from=2026-08-01&to=2026-08-31',
'/brokerage-accounts/baseline-0/events?from=2026-09-01&to=2026-10-31',
'/brokerage-accounts/all/events?from=2026-09-01&to=2026-10-31',
'/stocks?query=SYNTH',
'/bonds?query=SYNTH',
...[
['stocks', instruments[0].uid, 'dividends'],
['bonds', instruments[1].uid, 'coupons'],
].flatMap(([kind, uid, payout]) => [
`/${kind}/${uid}`,
`/${kind}/${uid}/quote`,
`/${kind}/${uid}/${kind === 'stocks' ? 'company' : 'metrics'}`,
`/${kind}/${uid}/candles?period=year`,
`/${kind}/${uid}/candles?period=all`,
`/${kind}/${uid}/${payout}`,
]),
];
async function settled() {
for (let index = 0; index < 6000; index++) {
await delay(10);
if (
!(await app
.get(PrismaService)
.tbankSyncWork.count({ where: { state: 'running' } }))
) {
return;
}
}
throw new Error('Import did not settle');
}
async function fresh() {
const db = app.get(PrismaService);
const fetchedAt = new Date();
await db.tbankConnection.updateMany({
data: { membershipAt: fetchedAt },
});
await db.tbankPortfolio.updateMany({ data: { fetchedAt } });
await db.tbankInstrumentSource.updateMany({ data: { fetchedAt } });
await db.tbankCandleCoverage.updateMany({
where: { isComplete: false },
data: { fetchedAt },
});
}
for (let pass = 0; pass < 3; pass++) {
for (const route of routes) {
await request(app.getHttpServer())
.get(`/api/v1${route}`)
.set('Authorization', `Bearer ${token}`)
.expect(200);
await settled();
}
await fresh();
}
const warmupRpc = { ...rpc };
// eslint-disable-next-line security/detect-object-injection -- Keys come from local test fixtures or numeric iteration, not external input.
for (const key of Object.keys(rpc)) rpc[key] = 0;
await app.close();
await start();
upstreamMode = 'failed';
const evidence = [];
const responses: Record<string, unknown> = {};
for (const route of routes) {
const read = async () => {
const start = performance.now();
const response = await request(app.getHttpServer())
.get(`/api/v1${route}`)
.set('Authorization', `Bearer ${token}`)
.expect(200);
const body = response.body as {
data: unknown;
meta: { availability: string };
};
if (route.includes('candles?period=all')) {
// eslint-disable-next-line jest/no-conditional-expect -- The parameterized fixture chooses the scenario; each branch asserts its outcome and the test also asserts common invariants.
expect((body.data as { candles: unknown[] }).candles).toHaveLength(
5000,
);
}
expect({ route, availability: body.meta.availability }).toEqual({
route,
availability: 'complete',
});
// eslint-disable-next-line security/detect-object-injection -- Keys come from local test fixtures or numeric iteration, not external input.
if (!responses[route]) {
// eslint-disable-next-line security/detect-object-injection -- Keys come from local test fixtures or numeric iteration, not external input.
responses[route] = body;
// eslint-disable-next-line security/detect-non-literal-fs-filename -- Paths are under an exclusive mkdtemp directory or the local migrations directory.
writeFileSync(
join(evidenceDirectory, 'browser-responses.json'),
JSON.stringify(responses),
);
}
return {
ms: performance.now() - start,
bytes: Buffer.byteLength(response.text),
};
};
await fresh();
const cold = await read();
serverTimes.length = 0;
const before = { ...rpc };
const samples = [];
for (let batch = 0; batch < 10; batch += 1) {
samples.push(...(await Promise.all(Array.from({ length: 10 }, read))));
}
const clientTimes = samples
.map((sample) => sample.ms)
.sort((a, b) => a - b);
const times = [...serverTimes].sort((a, b) => a - b);
expect(times).toHaveLength(100);
evidence.push({
route,
cold,
count: samples.length,
clientP95: clientTimes[94],
p50: times[49],
p95: times[94],
p99: times[98],
max: times[99],
maxBytes: Math.max(...samples.map((sample) => sample.bytes)),
rpc: Object.fromEntries(
Object.entries(rpc).map(([key, count]) => [
key,
// eslint-disable-next-line security/detect-object-injection -- Keys come from local test fixtures or numeric iteration, not external input.
count - (before[key] ?? 0),
]),
),
});
}
const result = {
phase: 'persistent',
runtime: process.version,
hardware: {
cpuQuota:
process.platform === 'linux'
? readFileSync('/sys/fs/cgroup/cpu.max', 'utf8').trim()
: null,
memoryLimit:
process.platform === 'linux'
? readFileSync('/sys/fs/cgroup/memory.max', 'utf8').trim()
: null,
cpu: cpus()[0]?.model,
cpus: cpus().length,
memoryBytes: totalmem(),
},
profile: {
accounts: 3,
operationsPerAccount: 100_000,
instruments: 200,
candlesPerInstrument: 5000,
upstream:
'synthetic failing upstream after population; transport limiter excluded',
timing:
'backend middleware ingress to response finish; client p95 separately',
limitation:
'Synthetic transport boundary; cgroup fields report constraints when run in Docker. Cross-host latency is not compared to baseline.',
},
evidence,
totalRpc: { ...rpc },
warmupRpc,
// eslint-disable-next-line security/detect-non-literal-fs-filename -- Paths stay in runner-created temporary directories or explicitly selected local test builds.
databaseBytes: statSync(
String(process.env.DATABASE_URL).replace('file:', ''),
).size,
};
const baseline = JSON.parse(
readFileSync('test/fixtures/tbank-read-baseline.json', 'utf8'),
) as { totalRpc: Record<string, number> };
const oldRpc = Object.values(baseline.totalRpc).reduce(
(sum, n) => sum + n,
0,
);
const newRpc = Object.values(rpc).reduce((sum, n) => sum + n, 0);
const output = {
...result,
oldRpc,
newRpc,
reduction: 1 - newRpc / oldRpc,
};
// eslint-disable-next-line security/detect-non-literal-fs-filename -- Paths are under an exclusive mkdtemp directory or the local migrations directory.
writeFileSync(
join(evidenceDirectory, 'persistent-budget.json'),
JSON.stringify(output, null, 2),
);
console.log(JSON.stringify({ ...output, evidenceDirectory }));
// Hold the source until HTTP reads finish: proves independence without a
// wall-clock race against the runner's throughput. The p95 gate stays below.
await fresh();
await app.get(PrismaService).tbankPortfolio.updateMany({
where: { accountId: 'baseline-0' },
data: { fetchedAt: new Date(0) },
});
const beforeStale = { ...rpc };
upstreamMode = 'delayed';
sourceGate = new Promise<void>((resolve) => {
releaseSource = resolve;
});
const startStale = performance.now();
try {
await Promise.all(
Array.from({ length: 20 }, () =>
request(app.getHttpServer())
.get('/api/v1/brokerage-accounts/baseline-0/portfolio')
.set('Authorization', `Bearer ${token}`)
.timeout(10_000)
.expect(200),
),
);
} finally {
releaseSource();
}
const staleResponsesMs = performance.now() - startStale;
await settled();
expect(rpc.portfolio - beforeStale.portfolio).toBe(1);
expect(rpc.positions - beforeStale.positions).toBe(1);
await app
.get(PrismaService)
.tbankPortfolio.deleteMany({ where: { accountId: 'baseline-0' } });
sourceGate = new Promise<void>((resolve) => {
releaseSource = resolve;
});
const coldStart = performance.now();
try {
const missing = await request(app.getHttpServer())
.get('/api/v1/brokerage-accounts/baseline-0/portfolio')
.set('Authorization', `Bearer ${token}`)
.timeout(10_000)
.expect(200);
expect((missing.body as { data: unknown }).data).toBeNull();
} finally {
releaseSource();
}
const coldImportMs = performance.now() - coldStart;
await settled();
Object.assign(output, {
staleResponsesMs,
staleLogicalPass: { portfolio: 1, positions: 1, retries: 0 },
coldImportMs,
});
// eslint-disable-next-line security/detect-non-literal-fs-filename -- Paths are under an exclusive mkdtemp directory or the local migrations directory.
writeFileSync(
join(evidenceDirectory, 'persistent-budget.json'),
JSON.stringify(output, null, 2),
);
expect(newRpc).toBe(0);
expect(evidence.filter((group) => group.p95 > 500)).toEqual([]);
expect(await app.get(PrismaService).tbankOperation.count()).toBe(300_000);
const idleRpc = { ...rpc };
await delay(1100);
expect(rpc).toEqual(idleRpc);
}, 300_000);
});