feat!: persist T-Bank data and synchronize reads
Backend security / security (push) Successful in 8s
Backend verify / verify (push) Failing after 10m45s
Conventional commits / commits (pull_request) Successful in 29s
Backend security / security (pull_request) Successful in 8s
Backend verify / verify (pull_request) Failing after 10m37s

Store account membership, portfolio, operations, forecasts and market sources in SQLite.
Coalesce scoped refreshes, fence connection revisions, preserve complete financial
snapshots and stabilize operation period state.

BREAKING CHANGE: investment reads return data/meta resource envelopes and locally
persisted snapshots; consumers must use the matching OpenAPI contract and explicit
synchronization endpoints.
This commit is contained in:
2026-09-26 21:29:15 +03:00
parent 0862d56c4e
commit d6e6c4ace5
116 changed files with 18321 additions and 5409 deletions
@@ -0,0 +1,294 @@
import { accountHistory } from '../../investment-sync-plan';
import { Injectable, NotFoundException } from '@nestjs/common';
import { randomUUID } from 'node:crypto';
import { MILLISECONDS_PER_SECOND } from '../../../../common/time';
import { StructuredLogger } from '../../../../common/logging/structured-logger.service';
import {
TbankAccessService,
TbankClient,
} from '../../../../integrations/tbank/tbank.api';
import {
BrokerageOperationsRepository,
type OperationAttempt,
} from '../infrastructure/brokerage-operations.repository';
import {
operationIntervals,
planOperationRead,
type OperationInterval,
uncoveredOperationIntervals,
} from '../domain/operations-sync-plan';
const operationsPageSize = 1000;
const priorities = { screen: 1, history: 2 } as const;
@Injectable()
export class BrokerageOperationsSyncService {
private readonly processRunId = randomUUID();
private readonly runningAttempts = new Set<string>();
private readonly ready = new Map<
string,
{ attempt: OperationAttempt; priority: number }
>();
private draining = false;
// Only short DB reservations/publications are serialized, never RPCs.
private reservations: Promise<unknown> = Promise.resolve();
constructor(
private readonly repository: BrokerageOperationsRepository,
private readonly access: TbankAccessService,
private readonly tbank: TbankClient,
private readonly logger: StructuredLogger,
) {}
async readPlan(
userId: string,
intervals: OperationInterval[],
options: {
movingEnd?: boolean;
force?: boolean;
observe?: boolean;
history?: boolean;
revision?: string;
} = {},
) {
const connection = await this.access.local(userId);
if (
!connection.membershipAt ||
(options.revision && options.revision !== connection.revision)
)
return { plan: null, handle: null };
if (
intervals.some(
(interval) =>
!connection.accounts.some(
(account) => account.accountId === interval.accountId,
),
)
)
throw new NotFoundException();
const coverage = await this.repository.coverage(
userId,
connection.revision,
intervals.map((interval) => interval.accountId),
);
const now = new Date();
const movingTo = Math.max(
...intervals.map((interval) => interval.to.valueOf()),
);
const targets = intervals.map((interval) =>
options.movingEnd && interval.to.valueOf() === movingTo
? { ...interval, to: now }
: interval,
);
const plan = planOperationRead(targets, coverage, now, options);
const complete = plan.intervals.every(
(interval) => interval.availability === 'complete',
);
const handle =
!options.observe && plan.load.length
? await this.request(
userId,
plan.load,
complete || Boolean(options.force),
'screen',
connection.revision,
)
: null;
if (!options.observe && !options.force && options.history) {
const accountIds = new Set(
intervals.map((interval) => interval.accountId),
);
const history = accountHistory(
connection.accounts.filter((account) =>
accountIds.has(account.accountId),
),
now,
);
const missing = planOperationRead(history, coverage, now, {
movingEnd: true,
}).load;
const remaining = uncoveredOperationIntervals(missing, targets);
if (remaining.length)
await this.request(
userId,
remaining,
false,
'history',
connection.revision,
);
}
return { plan, handle };
}
async request(
userId: string,
intervals: OperationInterval[],
atomic: boolean,
priority: keyof typeof priorities = 'screen',
revision?: string,
): Promise<string | null> {
const connection = await this.access.local(userId);
if (
connection.membershipAt &&
intervals.some(
(interval) =>
!connection.accounts.some(
(account) => account.accountId === interval.accountId,
),
)
)
throw new NotFoundException();
if (revision && revision !== connection.revision) return null;
const cutoff = new Date();
const bounded = operationIntervals(intervals)
.map((interval) => ({
...interval,
to: new Date(Math.min(interval.to.valueOf(), cutoff.valueOf())),
}))
.filter((interval) => interval.from < interval.to);
if (!bounded.length) return null;
const reserve = this.reservations.then(async () => {
const requested = await this.repository.request(
userId,
connection.revision,
operationIntervals(bounded),
this.processRunId,
atomic,
this.runningAttempts,
priorities[priority],
);
if (!requested) return null;
const { id, attemptId, revision } = requested.work;
this.runningAttempts.add(attemptId);
this.ready.set(attemptId, {
attempt: { id, attemptId, userId, revision },
priority: requested.work.priority,
});
if (!this.draining) {
this.draining = true;
// Startup and observation never enter this path.
setImmediate(() => {
void this.drain();
});
}
return id;
});
this.reservations = reserve.catch(() => undefined);
return reserve;
}
async observe(userId: string, id: string) {
const connection = await this.access.local(userId);
const work = await this.repository.observe(userId, connection.revision, id);
if (!work) throw new NotFoundException();
return work;
}
private async drain(): Promise<void> {
try {
while (this.ready.size) {
const selected = [...this.ready.values()].sort(
(a, b) => a.priority - b.priority,
)[0];
const { attempt } = selected;
this.ready.delete(attempt.attemptId);
let pending = false;
try {
pending = await this.step(attempt);
} catch {
this.logger.event(
'error',
'tbank.operations.sync.persistence_failed',
{ userId: attempt.userId, code: 'TBANK_UNAVAILABLE' },
);
}
const next = pending ? await this.repository.pending(attempt) : null;
if (next) {
const joined = this.ready.get(attempt.attemptId);
this.ready.set(attempt.attemptId, {
attempt,
priority: Math.min(
next.priority,
joined?.priority ?? next.priority,
),
});
} else {
this.ready.delete(attempt.attemptId);
this.runningAttempts.delete(attempt.attemptId);
}
}
} finally {
this.draining = false;
}
}
private async step(attempt: OperationAttempt): Promise<boolean> {
try {
const plan = await this.repository.pending(attempt);
if (!plan) return false;
if (!plan.next) {
const publish = this.reservations.then(async () => {
const current = await this.repository.pending(attempt);
if (!current) return false;
if (current.next) return true;
const published = await this.repository.publish(
attempt,
current.intervals,
new Date(),
true,
);
return published && Boolean(await this.repository.pending(attempt));
});
this.reservations = publish.catch(() => undefined);
return await publish;
}
const { interval, cursor } = plan.next;
const token = await this.access.tokenForAttempt(attempt);
if (!token) return false;
const response = await this.tbank.getOperationsByCursor(token, {
accountId: interval.accountId,
// Upstream inclusivity is not assumed; storage selects exact [from,to).
from: {
seconds: String(
Math.floor(interval.from.valueOf() / MILLISECONDS_PER_SECOND) - 1,
),
},
to: {
seconds: String(
Math.ceil(interval.to.valueOf() / MILLISECONDS_PER_SECOND) + 1,
),
},
...(interval.instrumentUid
? { instrumentId: interval.instrumentUid }
: {}),
cursor,
limit: operationsPageSize,
state: 'OPERATION_STATE_EXECUTED',
});
if (
!(await this.repository.stagePage(attempt, interval, cursor, response))
)
return false;
if (!plan.atomic && !response.hasNext) {
if (
!(await this.repository.publish(
attempt,
[interval],
new Date(),
true,
))
)
return false;
}
// One page per turn: newly requested screen work can overtake backfill.
// TbankClient alone owns the bounded RPC queue and retries.
return true;
} catch (error) {
await this.repository.fail(
attempt,
error instanceof Error && error.message === 'INVALID_TBANK_TOKEN',
);
return false;
}
}
}
@@ -1,373 +0,0 @@
import {
BadRequestException,
ConflictException,
NotFoundException,
ServiceUnavailableException,
} from '@nestjs/common';
jest.mock('../../../../integrations/tbank/tbank-access.service', () => ({
TbankAccessService: class TbankAccessService {},
}));
import type { TbankAccountContext } from '../../../../integrations/tbank/tbank-access.service';
import { BrokerageOperationsService } from './brokerage-operations.service';
describe('BrokerageOperationsService', () => {
const access = { withAccount: jest.fn() };
const tbank = {
getOperationsByCursor: jest.fn(),
};
const requestContext = { set: jest.fn() };
const metadata = { resolve: jest.fn() };
const service = new BrokerageOperationsService(
access as never,
tbank as never,
requestContext as never,
metadata as never,
);
let operationsRequest: unknown;
let generation: string;
beforeEach(() => {
jest.clearAllMocks();
access.withAccount.mockReset();
tbank.getOperationsByCursor.mockReset();
metadata.resolve.mockReset();
generation = 'generation-1';
access.withAccount.mockImplementation(
(
_userId: string,
_accountId: string,
handler: (context: TbankAccountContext) => Promise<unknown>,
) =>
handler({
token: 'token',
generation,
account: { id: 'account', type: 'ACCOUNT_TYPE_TINKOFF', name: '' },
}),
);
tbank.getOperationsByCursor.mockImplementation(
(_token: string, request: unknown) => {
operationsRequest = request;
return Promise.resolve({ items: [] });
},
);
(service as unknown as { pages: Map<string, unknown> }).pages.clear();
});
afterEach(() => {
jest.useRealTimers();
jest.restoreAllMocks();
});
it('uses the current UTC year by default and asks only for executed operations', async () => {
await service.get('user', 'account', {});
const request = operationsRequest as {
state: string;
from: { seconds: string };
};
expect(request.state).toBe('OPERATION_STATE_EXECUTED');
expect(request.from.seconds).toBe(
String(Math.floor(Date.UTC(new Date().getUTCFullYear(), 0, 1) / 1000)),
);
});
it('uses inclusive date boundaries, filters categories, and preserves cursors', async () => {
tbank.getOperationsByCursor.mockResolvedValue({
hasNext: true,
nextCursor: 'next',
items: [
{ id: 'income', date: { seconds: '1' }, type: 'OPERATION_TYPE_COUPON' },
{
id: 'fee',
date: { seconds: '2' },
type: 'OPERATION_TYPE_BROKER_FEE',
},
],
});
await expect(
service.get('user', 'account', {
from: '2025-01-01',
to: '2025-01-31',
categories: ['income'],
cursor: 'previous',
limit: 10,
}),
).resolves.toMatchObject({
items: [{ id: 'income' }],
nextCursor: 'next',
hasNext: true,
});
expect(tbank.getOperationsByCursor).toHaveBeenCalledWith(
'token',
expect.objectContaining({
from: { seconds: '1735689600' },
to: { seconds: '1738367999', nanos: 999_999_999 },
cursor: 'previous',
}),
);
});
it('includes resolved instrument logo metadata in a ledger entry', async () => {
tbank.getOperationsByCursor.mockResolvedValue({
items: [
{
id: 'coupon',
date: { seconds: '1' },
type: 'OPERATION_TYPE_COUPON',
instrumentUid: 'instrument-uid',
},
],
});
metadata.resolve.mockResolvedValue({
brand: { logoName: 'instrument-logo.png' },
});
await expect(service.get('user', 'account', {})).resolves.toMatchObject({
items: [{ id: 'coupon', logoName: 'instrument-logo.png' }],
});
});
it('resolves repeated instrument metadata only once per ledger page', async () => {
tbank.getOperationsByCursor.mockResolvedValue({
items: [
{
id: 'first',
date: { seconds: '1' },
type: 'OPERATION_TYPE_COUPON',
instrumentUid: 'instrument-uid',
},
{
id: 'second',
date: { seconds: '2' },
type: 'OPERATION_TYPE_COUPON',
instrumentUid: 'instrument-uid',
},
],
});
metadata.resolve.mockResolvedValue({
brand: { logoName: 'instrument-logo.png' },
});
await service.get('user', 'account', {});
expect(metadata.resolve).toHaveBeenCalledTimes(1);
});
it('does not let a UID-only lookup suppress a later FIGI fallback on the same page', async () => {
tbank.getOperationsByCursor.mockResolvedValue({
items: [
{
id: 'first',
date: { seconds: '1' },
type: 'OPERATION_TYPE_COUPON',
instrumentUid: 'instrument-uid',
},
{
id: 'second',
date: { seconds: '2' },
type: 'OPERATION_TYPE_COUPON',
instrumentUid: 'instrument-uid',
figi: 'BBG000000001',
},
],
});
metadata.resolve.mockResolvedValueOnce(null).mockResolvedValueOnce({
brand: { logoName: 'fallback-logo.png' },
});
await expect(service.get('user', 'account', {})).resolves.toMatchObject({
items: [
{ id: 'first', logoName: null },
{ id: 'second', logoName: 'fallback-logo.png' },
],
});
expect(metadata.resolve).toHaveBeenNthCalledWith(
1,
'token',
'instrument-uid',
null,
);
expect(metadata.resolve).toHaveBeenNthCalledWith(
2,
'token',
'instrument-uid',
'BBG000000001',
);
});
it('returns an empty page without calling T-Bank when the period starts in the future', async () => {
await expect(
service.get('user', 'account', { from: '2999-01-01T00:00:00.000Z' }),
).resolves.toMatchObject({
accountId: 'account',
items: [],
nextCursor: null,
hasNext: false,
});
expect(tbank.getOperationsByCursor).not.toHaveBeenCalled();
});
it('rejects inaccessible accounts before reading operations', async () => {
access.withAccount.mockRejectedValueOnce(new NotFoundException());
await expect(service.get('user', 'account', {})).rejects.toEqual(
expect.any(NotFoundException),
);
expect(tbank.getOperationsByCursor).not.toHaveBeenCalled();
});
it('retains existing T-Bank error semantics', async () => {
access.withAccount.mockRejectedValueOnce(
new BadRequestException({ code: 'INVALID_TBANK_TOKEN' }),
);
await expect(service.get('user', 'account', {})).rejects.toEqual(
expect.any(BadRequestException),
);
tbank.getOperationsByCursor.mockRejectedValueOnce(
new ServiceUnavailableException({ code: 'TBANK_UNAVAILABLE' }),
);
await expect(service.get('user', 'account', {})).rejects.toEqual(
expect.any(ServiceUnavailableException),
);
});
it('reuses one fresh page for equivalent categories within the default end-time bucket', async () => {
jest.useFakeTimers();
jest.setSystemTime(new Date('2026-08-23T12:00:00.100Z'));
await service.get('user', 'account', {
categories: ['tax', 'income', 'income'],
});
jest.advanceTimersByTime(500);
await service.get('user', 'account', { categories: ['income', 'tax'] });
expect(tbank.getOperationsByCursor).toHaveBeenCalledTimes(1);
});
it('does not let a cached page bypass account ownership checks', async () => {
const query = { from: '2025-01-01', to: '2025-01-31' };
await service.get('user', 'account', query);
access.withAccount.mockRejectedValueOnce(new NotFoundException());
await expect(service.get('user', 'account', query)).rejects.toEqual(
expect.any(NotFoundException),
);
expect(tbank.getOperationsByCursor).toHaveBeenCalledTimes(1);
});
it('does not reuse a cached ledger page across connection generations', async () => {
const query = { from: '2025-01-01', to: '2025-01-31' };
await service.get('user', 'account', query);
await service.get('user', 'account', query);
generation = 'generation-2';
await service.get('user', 'account', query);
expect(tbank.getOperationsByCursor).toHaveBeenCalledTimes(2);
});
it('does not reuse a cached ledger page after the connection is deleted', async () => {
const query = { from: '2025-01-01', to: '2025-01-31' };
await service.get('user', 'account', query);
access.withAccount.mockRejectedValueOnce(
new ConflictException({ code: 'TBANK_NOT_CONNECTED' }),
);
await expect(service.get('user', 'account', query)).rejects.toEqual(
expect.any(ConflictException),
);
expect(tbank.getOperationsByCursor).toHaveBeenCalledTimes(1);
});
it('keeps a late previous-generation page unavailable to ledger and income collection', async () => {
let finishPrevious: ((value: unknown) => void) | undefined;
const previous = new Promise((resolve) => {
finishPrevious = resolve;
});
tbank.getOperationsByCursor
.mockReturnValueOnce(previous)
.mockResolvedValueOnce({
items: [operation('current')],
});
const range = { from: '2025-01-01', to: '2025-01-31' };
const previousRequest = service.collectRanges('user', 'account', [range]);
generation = 'generation-2';
const current = await service.collectRanges('user', 'account', [range]);
finishPrevious?.({ items: [operation('previous')] });
await previousRequest;
const reused = await service.collectRanges('user', 'account', [range]);
expect(current.pages[0][0].items[0].id).toBe('current');
expect(reused.pages[0][0].items[0].id).toBe('current');
expect(tbank.getOperationsByCursor).toHaveBeenCalledTimes(2);
});
it('collects every cursor page and rejects a malformed continuation cursor', async () => {
tbank.getOperationsByCursor
.mockResolvedValueOnce({ hasNext: true, nextCursor: 'next', items: [] })
.mockResolvedValueOnce({ hasNext: false, items: [] });
await expect(
service.collectRanges('user', 'account', [
{ from: '2026-01-01', to: '2026-01-31' },
]),
).resolves.toMatchObject({
pages: [[{ hasNext: true }, { hasNext: false }]],
});
tbank.getOperationsByCursor.mockReset();
tbank.getOperationsByCursor.mockResolvedValue({
hasNext: true,
nextCursor: 'same',
items: [],
});
await expect(
service.collectRanges('user', 'account', [
{ from: '2026-02-01', to: '2026-02-28' },
]),
).rejects.toMatchObject({ response: { code: 'TBANK_UNAVAILABLE' } });
});
it('collects an in-progress Moscow day only through the current instant', async () => {
jest.useFakeTimers();
jest.setSystemTime(new Date('2026-08-27T21:44:18.510Z'));
await service.collectRanges('user', 'account', [
{ from: '2026-08-01', to: '2026-08-28' },
]);
expect(tbank.getOperationsByCursor).toHaveBeenCalledWith(
'token',
expect.objectContaining({
from: { seconds: '1785531600' },
to: { seconds: '1787867058' },
limit: 1000,
}),
);
});
it('evicts the least recently used ledger page after one hundred entries', async () => {
for (let index = 0; index < 100; index += 1)
await service.get('user', 'account', { cursor: String(index) });
await service.get('user', 'account', { cursor: '0' });
await service.get('user', 'account', { cursor: '100' });
await service.get('user', 'account', { cursor: '1' });
const pages = (service as unknown as { pages: { size: number } }).pages;
expect(pages.size).toBe(100);
expect(tbank.getOperationsByCursor).toHaveBeenCalledTimes(102);
});
});
function operation(id: string) {
return {
id,
date: { seconds: '1' },
type: 'OPERATION_TYPE_COUPON',
};
}
File diff suppressed because it is too large Load Diff