feat!: persist T-Bank data and synchronize reads

Store account membership, portfolio, operations, forecasts and market sources in SQLite. Coalesce scoped refreshes, fence connection revisions, preserve complete financial snapshots and stabilize operation period state. BREAKING CHANGE: investment reads return data/meta resource envelopes and locally persisted snapshots; consumers must use the matching OpenAPI contract and explicit synchronization endpoints.
2026-09-26 21:29:15 +03:00
parent 0862d56c4e
commit d6e6c4ace5
116 changed files with 18321 additions and 5409 deletions
@@ -0,0 +1,569 @@
+import { INestApplication } from '@nestjs/common';
+import { JwtService } from '@nestjs/jwt';
+import { Test } from '@nestjs/testing';
+import type { Server } from 'node:http';
+import { cpus, totalmem } from 'node:os';
+import { writeFileSync, statSync, readFileSync } from 'node:fs';
+import { setTimeout as delay } from 'node:timers/promises';
+import type { Request, Response, NextFunction } from 'express';
+import { performance } from 'node:perf_hooks';
+import request from 'supertest';
+import { AppModule } from '../src/app.module';
+import {
+  TbankClient,
+  TbankOperationsByCursorRequest,
+} from '../src/integrations/tbank/tbank.client';
+import { TbankTokenService } from '../src/integrations/tbank/tbank-token.service';
+import { configureApp } from '../src/main';
+import { PrismaService } from '../src/prisma/prisma.service';
+import { MoexClient } from '../src/integrations/moex/moex.api';
+import { NcrRatingsClient } from '../src/integrations/ncr/ncr.api';
+
+// Same navigation and synthetic sources as the saved baseline; isolated SQLite only.
+describe('T-Bank persisted read budget', () => {
+  let app: INestApplication<Server>;
+  const owner = 'synthetic-baseline-owner';
+  const accounts = Array.from({ length: 3 }, (_, index) => ({
+    id: `baseline-${index}`,
+    name: `Synthetic ${index}`,
+    type: 'ACCOUNT_TYPE_TINKOFF',
+    openedDate: { seconds: Date.UTC(2016, 8, 1) / 1000 },
+  }));
+  const instruments = Array.from({ length: 200 }, (_, index) => ({
+    uid: `00000000-0000-4000-8000-${String(index).padStart(12, '0')}`,
+    ticker: `SYNTH${index}`,
+    name: `Synthetic instrument ${index}`,
+    assetUid: `asset-${index}`,
+    isin: `SYNTHETIC${index}`,
+    currency: 'rub',
+    instrumentKind:
+      index % 2 ? 'INSTRUMENT_TYPE_BOND' : 'INSTRUMENT_TYPE_SHARE',
+    classCode: index % 2 ? 'TQOB' : 'TQBR',
+    apiTradeAvailableFlag: true,
+    first_1dayCandleDate: { seconds: Date.UTC(2026, 7, 31 - 4999) / 1000 },
+    nominal: { currency: 'rub', units: '1000' },
+    aciValue: { currency: 'rub', units: '12', nano: 345678900 },
+  }));
+  const candles = Array.from({ length: 5000 }, (_, index) => ({
+    time: { seconds: Date.UTC(2026, 7, 31 - 4999 + index) / 1000 },
+    close: { units: '100', nano: index },
+    isComplete: true,
+  }));
+  const operations = Array.from({ length: 100_000 }, (_, index) => {
+    // 1,000 measured-month rows, 200 income receipts, remaining rows over 10 years.
+    const time =
+      index < 1000
+        ? Date.UTC(2026, 7, 1) + index * 2_500_000
+        : Date.UTC(2016, 8, 1) + (index - 1000) * 3_160_000;
+    const instrument = instruments[index % instruments.length];
+    return {
+      id: `operation-${index}`,
+      date: { seconds: String(Math.floor(time / 1000)) },
+      type: index % 5 === 0 ? 'OPERATION_TYPE_COUPON' : 'OPERATION_TYPE_BUY',
+      instrumentUid: instrument.uid,
+      ticker: instrument.ticker,
+      instrumentType: 'bond',
+      payment: { currency: 'rub', units: '123', nano: 123456789 },
+      quantity: '1',
+      quantityDone: '1',
+    };
+  }).sort((a, b) => Number(b.date.seconds) - Number(a.date.seconds));
+  const rpc: Record<string, number> = {};
+  let upstreamMode: 'ready' | 'delayed' | 'failed' = 'ready';
+  const serverTimes: number[] = [];
+  async function counted<T>(name: string, value: T): Promise<T> {
+    rpc[name] = (rpc[name] ?? 0) + 1;
+    if (upstreamMode === 'delayed') await delay(2000);
+    if (upstreamMode === 'failed')
+      return Promise.reject(new Error('Synthetic outage'));
+    return Promise.resolve(value);
+  }
+  const upstream = {
+    getOpenAccounts: () => counted('accounts', accounts),
+    getPortfolio: () =>
+      counted('portfolio', {
+        positions: instruments.map((instrument, index) => ({
+          instrumentUid: instrument.uid,
+          instrumentType: index % 2 ? 'bond' : 'share',
+          quantity: { units: '1' },
+          currentPrice: { currency: 'rub', units: '1000' },
+        })),
+      }),
+    getPositions: () =>
+      counted('positions', { money: [{ currency: 'rub', units: '100' }] }),
+    getInstrumentByUid: (_token: string, uid: string) =>
+      counted(
+        'metadata',
+        instruments.find((instrument) => instrument.uid === uid),
+      ),
+    findShares: (_token: string, query: string) =>
+      counted(
+        'findShares',
+        instruments.filter(
+          (item) =>
+            item.instrumentKind === 'INSTRUMENT_TYPE_SHARE' &&
+            item.ticker.includes(query),
+        ),
+      ),
+    findBonds: (_token: string, query: string) =>
+      counted(
+        'findBonds',
+        instruments.filter(
+          (item) =>
+            item.instrumentKind === 'INSTRUMENT_TYPE_BOND' &&
+            item.ticker.includes(query),
+        ),
+      ),
+    getShare: (_token: string, uid: string) =>
+      counted(
+        'share',
+        instruments.find((item) => item.uid === uid),
+      ),
+    getBond: (_token: string, uid: string) =>
+      counted(
+        'bond',
+        instruments.find((item) => item.uid === uid),
+      ),
+    getLastPrice: (_token: string, uid: string) =>
+      counted('lastPrice', {
+        instrumentUid: uid,
+        price: { units: '100' },
+        time: { seconds: Date.UTC(2026, 7, 31) / 1000 },
+      }),
+    getLastPrices: (_token: string, uids: string[]) =>
+      counted(
+        'lastPrices',
+        uids.map((uid) => ({ instrumentUid: uid, price: { units: '100' } })),
+      ),
+    getTradingStatus: () =>
+      counted('tradingStatus', 'SECURITY_TRADING_STATUS_NORMAL_TRADING'),
+    getCandles: (_token: string, _uid: string, from: Date, to: Date) =>
+      counted(
+        'candles',
+        candles.filter(
+          (row) =>
+            row.time.seconds * 1000 >= from.valueOf() &&
+            row.time.seconds * 1000 <= to.valueOf(),
+        ),
+      ),
+    getAssetDescription: () => counted('assetDescription', 'Synthetic company'),
+    getFundamentals: () =>
+      counted('fundamentals', {
+        currency: 'rub',
+        revenueTtm: 1_000_000,
+        peRatioTtm: 5,
+      }),
+    getDividends: () =>
+      counted('dividends', [
+        {
+          dividendNet: { currency: 'rub', units: '10' },
+          recordDate: { seconds: Date.UTC(2026, 9, 1) / 1000 },
+        },
+      ]),
+    getBondCoupons: () =>
+      counted('coupons', [
+        {
+          payOneBond: { currency: 'rub', units: '10' },
+          couponDate: { seconds: Date.UTC(2026, 9, 1) / 1000 },
+          fixDate: { seconds: Date.UTC(2026, 8, 30) / 1000 },
+          couponPeriod: 30,
+        },
+      ]),
+    getOperationsByCursor: (
+      _token: string,
+      query: TbankOperationsByCursorRequest,
+    ) => {
+      const rows = operations.filter(
+        (row) =>
+          Number(row.date.seconds) >= Number(query.from?.seconds ?? 0) &&
+          Number(row.date.seconds) <= Number(query.to?.seconds ?? Infinity),
+      );
+      const offset = Number(query.cursor ?? 0);
+      const end = offset + (query.limit ?? 50);
+      return counted('operations', {
+        items: rows.slice(offset, end),
+        hasNext: end < rows.length,
+        nextCursor: end < rows.length ? String(end) : '',
+      });
+    },
+  };
+  const token = new JwtService().sign(
+    { sub: owner, type: 'access' },
+    { secret: process.env.JWT_SECRET },
+  );
+
+  async function start() {
+    const module = await Test.createTestingModule({ imports: [AppModule] })
+      .overrideProvider(TbankClient)
+      .useValue(upstream)
+      .overrideProvider(MoexClient)
+      .useValue({
+        bondEvents: () =>
+          counted('moexEvents', [
+            {
+              type: 'coupon',
+              date: '2026-10-01',
+              value: '10',
+              currency: 'RUB',
+            },
+          ]),
+        bondYield: () => counted('moexYield', null),
+      })
+      .overrideProvider(NcrRatingsClient)
+      .useValue({ rating: () => counted('ncrRating', null) })
+      .compile();
+    app = module.createNestApplication();
+    app.use((_req: Request, res: Response, next: NextFunction) => {
+      const start = performance.now();
+      res.once('finish', () => serverTimes.push(performance.now() - start));
+      next();
+    });
+    configureApp(app);
+    await app.init();
+    await app.listen(0, '127.0.0.1');
+  }
+  beforeAll(async () => {
+    await start();
+    const prisma = app.get(PrismaService);
+    await prisma.user.create({
+      data: {
+        id: owner,
+        email: 'baseline@example.invalid',
+        passwordHash: 'unused',
+      },
+    });
+    const connection = await prisma.tbankConnection.create({
+      data: {
+        userId: owner,
+        encryptedToken: app.get(TbankTokenService).encrypt('synthetic-token'),
+        connectedAt: new Date(),
+        membershipAt: new Date(),
+        membershipRevision: 1,
+      },
+    });
+    const now = new Date();
+    for (const [index, account] of accounts.entries()) {
+      const scope = {
+        userId: owner,
+        revision: connection.revision,
+        accountId: account.id,
+      };
+      await prisma.tbankAccount.create({
+        data: {
+          ...scope,
+          name: account.name,
+          type: 'BROKERAGE',
+          openedOn: '2016-09-01',
+          upstreamOrder: index,
+          historyFrom: new Date('2016-09-01Z'),
+          historyBoundary: 'openedOn',
+        },
+      });
+      for (let offset = 0; offset < operations.length; offset += 1000)
+        await prisma.tbankOperation.createMany({
+          data: operations.slice(offset, offset + 1000).map((row) => ({
+            ...scope,
+            brokerOperationId: row.id,
+            occurredAt: new Date(Number(row.date.seconds) * 1000),
+            instrumentUid: row.instrumentUid,
+            category: row.type === 'OPERATION_TYPE_COUPON' ? 'income' : 'trade',
+            payload: row,
+          })),
+        });
+      await prisma.tbankOperationCoverage.create({
+        data: {
+          ...scope,
+          from: new Date('2016-08-31T21:00:00Z'),
+          to: now,
+          fetchedAt: now,
+          asOf: now,
+        },
+      });
+    }
+    for (const instrument of instruments) {
+      await prisma.tbankInstrument.create({
+        data: {
+          provider: 'tbank',
+          uid: instrument.uid,
+          ticker: instrument.ticker,
+          instrumentType:
+            instrument.instrumentKind === 'INSTRUMENT_TYPE_SHARE'
+              ? 'share'
+              : 'bond',
+          metadata: instrument,
+          fetchedAt: now,
+        },
+      });
+      await prisma.$executeRawUnsafe(
+        `WITH RECURSIVE days(n) AS (SELECT 0 UNION ALL SELECT n+1 FROM days WHERE n<4999)
+        INSERT INTO TbankCandle(provider,instrumentUid,interval,time,close,payload,isComplete,fetchedAt)
+        SELECT 'tbank',?,'day',strftime('%Y-%m-%dT%H:%M:%fZ', (?+n*86400000)/1000.0, 'unixepoch'),'100','{}',1,? FROM days`,
+        instrument.uid,
+        candles[0].time.seconds * 1000,
+        now.toISOString(),
+      );
+      const today = new Date(now);
+      today.setUTCHours(0, 0, 0, 0);
+      await prisma.tbankCandleCoverage.createMany({
+        data: [
+          {
+            provider: 'tbank',
+            instrumentUid: instrument.uid,
+            interval: 'day',
+            from: new Date(candles[0].time.seconds * 1000),
+            to: today,
+            isComplete: true,
+            fetchedAt: now,
+          },
+          {
+            provider: 'tbank',
+            instrumentUid: instrument.uid,
+            interval: 'day',
+            from: today,
+            to: new Date(today.valueOf() + 86400000),
+            isComplete: false,
+            fetchedAt: now,
+          },
+        ],
+      });
+    }
+    expect(await prisma.tbankOperation.count()).toBe(300_000);
+    expect(await prisma.tbankCandle.count()).toBe(1_000_000);
+  }, 300_000);
+  afterAll(async () => app?.close());
+
+  it('measures the same navigation with 10 clients and 100 reads per family', async () => {
+    expect(operations).toHaveLength(100_000);
+    expect(candles).toHaveLength(5000);
+    const routes = [
+      '/brokerage-accounts',
+      '/brokerage-accounts/baseline-0/portfolio',
+      '/brokerage-accounts/all/portfolio',
+      '/brokerage-accounts/baseline-0/operations?from=2026-08-01&to=2026-08-31&limit=50',
+      '/brokerage-accounts/all/operations?from=2026-08-01&to=2026-08-31&limit=50',
+      '/brokerage-accounts/baseline-0/income-analytics?from=2026-08-01&to=2026-08-31',
+      '/brokerage-accounts/all/income-analytics?from=2026-08-01&to=2026-08-31',
+      '/brokerage-accounts/baseline-0/events?from=2026-09-01&to=2026-10-31',
+      '/brokerage-accounts/all/events?from=2026-09-01&to=2026-10-31',
+      '/stocks?query=SYNTH',
+      '/bonds?query=SYNTH',
+      ...[
+        ['stocks', instruments[0].uid, 'dividends'],
+        ['bonds', instruments[1].uid, 'coupons'],
+      ].flatMap(([kind, uid, payout]) => [
+        `/${kind}/${uid}`,
+        `/${kind}/${uid}/quote`,
+        `/${kind}/${uid}/${kind === 'stocks' ? 'company' : 'metrics'}`,
+        `/${kind}/${uid}/candles?period=year`,
+        `/${kind}/${uid}/candles?period=all`,
+        `/${kind}/${uid}/${payout}`,
+      ]),
+    ];
+    async function settled() {
+      for (let index = 0; index < 6000; index++) {
+        await delay(10);
+        if (
+          !(await app
+            .get(PrismaService)
+            .tbankSyncWork.count({ where: { state: 'running' } }))
+        )
+          return;
+      }
+      throw new Error('Import did not settle');
+    }
+    async function fresh() {
+      const db = app.get(PrismaService);
+      const fetchedAt = new Date();
+      await db.tbankConnection.updateMany({
+        data: { membershipAt: fetchedAt },
+      });
+      await db.tbankPortfolio.updateMany({ data: { fetchedAt } });
+      await db.tbankInstrumentSource.updateMany({ data: { fetchedAt } });
+      await db.tbankCandleCoverage.updateMany({
+        where: { isComplete: false },
+        data: { fetchedAt },
+      });
+    }
+    for (let pass = 0; pass < 3; pass++) {
+      for (const route of routes) {
+        await request(app.getHttpServer())
+          .get(`/api/v1${route}`)
+          .set('Authorization', `Bearer ${token}`)
+          .expect(200);
+        await settled();
+      }
+      await fresh();
+    }
+    const warmupRpc = { ...rpc };
+    for (const key of Object.keys(rpc)) rpc[key] = 0;
+    await app.close();
+    await start();
+    upstreamMode = 'failed';
+    const evidence = [];
+    const responses: Record<string, unknown> = {};
+    for (const route of routes) {
+      const read = async () => {
+        const start = performance.now();
+        const response = await request(app.getHttpServer())
+          .get(`/api/v1${route}`)
+          .set('Authorization', `Bearer ${token}`)
+          .expect(200);
+        const body = response.body as {
+          meta: { availability: string };
+          data: unknown;
+        };
+        if (route.includes('candles?period=all'))
+          expect((body.data as { candles: unknown[] }).candles).toHaveLength(
+            5000,
+          );
+        expect({ route, availability: body.meta.availability }).toEqual({
+          route,
+          availability: 'complete',
+        });
+        if (!responses[route]) {
+          responses[route] = body;
+          writeFileSync(
+            '/tmp/wealthledger-browser-responses.json',
+            JSON.stringify(responses),
+          );
+        }
+        return {
+          ms: performance.now() - start,
+          bytes: Buffer.byteLength(response.text),
+        };
+      };
+      await fresh();
+      const cold = await read();
+      serverTimes.length = 0;
+      const before = { ...rpc };
+      const samples = [];
+      for (let batch = 0; batch < 10; batch += 1)
+        samples.push(...(await Promise.all(Array.from({ length: 10 }, read))));
+      const clientTimes = samples
+        .map((sample) => sample.ms)
+        .sort((a, b) => a - b);
+      const times = [...serverTimes].sort((a, b) => a - b);
+      expect(times).toHaveLength(100);
+      evidence.push({
+        route,
+        cold,
+        count: samples.length,
+        clientP95: clientTimes[94],
+        p50: times[49],
+        p95: times[94],
+        p99: times[98],
+        max: times[99],
+        maxBytes: Math.max(...samples.map((sample) => sample.bytes)),
+        rpc: Object.fromEntries(
+          Object.entries(rpc).map(([key, count]) => [
+            key,
+            count - (before[key] ?? 0),
+          ]),
+        ),
+      });
+    }
+    const result = {
+      phase: 'persistent',
+      runtime: process.version,
+      hardware: {
+        cpuQuota:
+          process.platform === 'linux'
+            ? readFileSync('/sys/fs/cgroup/cpu.max', 'utf8').trim()
+            : null,
+        memoryLimit:
+          process.platform === 'linux'
+            ? readFileSync('/sys/fs/cgroup/memory.max', 'utf8').trim()
+            : null,
+        cpu: cpus()[0]?.model,
+        cpus: cpus().length,
+        memoryBytes: totalmem(),
+      },
+      profile: {
+        accounts: 3,
+        operationsPerAccount: 100_000,
+        instruments: 200,
+        candlesPerInstrument: 5000,
+        upstream:
+          'synthetic failing upstream after population; transport limiter excluded',
+        timing:
+          'backend middleware ingress to response finish; client p95 separately',
+        limitation:
+          'Synthetic transport boundary; cgroup fields report constraints when run in Docker. Cross-host latency is not compared to baseline.',
+      },
+      evidence,
+      totalRpc: { ...rpc },
+      warmupRpc,
+      databaseBytes: statSync(
+        String(process.env.DATABASE_URL).replace('file:', ''),
+      ).size,
+    };
+    const baseline = JSON.parse(
+      readFileSync('test/fixtures/tbank-read-baseline.json', 'utf8'),
+    ) as { totalRpc: Record<string, number> };
+    const oldRpc = Object.values(baseline.totalRpc).reduce(
+      (sum, n) => sum + n,
+      0,
+    );
+    const newRpc = Object.values(rpc).reduce((sum, n) => sum + n, 0);
+    const output = {
+      ...result,
+      oldRpc,
+      newRpc,
+      reduction: 1 - newRpc / oldRpc,
+    };
+    writeFileSync(
+      '/tmp/wealthledger-persistent-budget.json',
+      JSON.stringify(output, null, 2),
+    );
+    console.log(JSON.stringify(output));
+    // Delayed dependency cannot hold a data response or multiply one stale pass.
+    await fresh();
+    await app.get(PrismaService).tbankPortfolio.updateMany({
+      where: { accountId: 'baseline-0' },
+      data: { fetchedAt: new Date(0) },
+    });
+    const beforeStale = { ...rpc };
+    upstreamMode = 'delayed';
+    const startStale = performance.now();
+    await Promise.all(
+      Array.from({ length: 20 }, () =>
+        request(app.getHttpServer())
+          .get('/api/v1/brokerage-accounts/baseline-0/portfolio')
+          .set('Authorization', `Bearer ${token}`)
+          .expect(200),
+      ),
+    );
+    const staleResponsesMs = performance.now() - startStale;
+    expect(staleResponsesMs).toBeLessThan(2000);
+    await settled();
+    expect(rpc.portfolio - beforeStale.portfolio).toBe(1);
+    expect(rpc.positions - beforeStale.positions).toBe(1);
+    await app
+      .get(PrismaService)
+      .tbankPortfolio.deleteMany({ where: { accountId: 'baseline-0' } });
+    const coldStart = performance.now();
+    const missing = await request(app.getHttpServer())
+      .get('/api/v1/brokerage-accounts/baseline-0/portfolio')
+      .set('Authorization', `Bearer ${token}`)
+      .expect(200);
+    expect((missing.body as { data: unknown }).data).toBeNull();
+    const coldImportMs = performance.now() - coldStart;
+    expect(coldImportMs).toBeLessThan(2000);
+    await settled();
+    Object.assign(output, {
+      staleResponsesMs,
+      staleLogicalPass: { portfolio: 1, positions: 1, retries: 0 },
+      coldImportMs,
+    });
+    writeFileSync(
+      '/tmp/wealthledger-persistent-budget.json',
+      JSON.stringify(output, null, 2),
+    );
+    expect(newRpc).toBe(0);
+    expect(evidence.filter((group) => group.p95 > 500)).toEqual([]);
+    expect(await app.get(PrismaService).tbankOperation.count()).toBe(300_000);
+    const idleRpc = { ...rpc };
+    await delay(1100);
+    expect(rpc).toEqual(idleRpc);
+  }, 300_000);
+});