feat!: persist T-Bank data and synchronize reads
Backend security / security (push) Successful in 8s
Backend verify / verify (push) Failing after 10m45s
Conventional commits / commits (pull_request) Successful in 29s
Backend security / security (pull_request) Successful in 8s
Backend verify / verify (pull_request) Failing after 10m37s

Store account membership, portfolio, operations, forecasts and market sources in SQLite.
Coalesce scoped refreshes, fence connection revisions, preserve complete financial
snapshots and stabilize operation period state.

BREAKING CHANGE: investment reads return data/meta resource envelopes and locally
persisted snapshots; consumers must use the matching OpenAPI contract and explicit
synchronization endpoints.
This commit is contained in:
2026-09-26 21:29:15 +03:00
parent 0862d56c4e
commit d6e6c4ace5
116 changed files with 18321 additions and 5409 deletions
@@ -0,0 +1,665 @@
import type { TbankDividend } from '../../../../integrations/tbank/tbank.api';
import type { MoexBondEvent } from '../../../../integrations/moex/moex.api';
import { CandleRepository } from '../infrastructure/candle.repository';
import { candleGaps, utcDay } from '../domain/candle-coverage';
import {
decimalFromString,
divide,
formatDecimal,
} from '../../../../common/decimal';
import { decimal } from '../../../../integrations/tbank/decimal';
import {
MILLISECONDS_PER_DAY,
MILLISECONDS_PER_SECOND,
} from '../../../../common/time';
import { MoexClient } from '../../../../integrations/moex/moex.api';
import { NcrRatingsClient } from '../../../../integrations/ncr/ncr.api';
import type {
TbankCandle,
TbankFundamentals,
} from '../../../../integrations/tbank/tbank.api';
import type { TbankCoupon } from '../../../../integrations/tbank/tbank-market.types';
import { previousClose, timestamp } from './instrument-market';
import { Injectable, NotFoundException } from '@nestjs/common';
import { randomUUID } from 'node:crypto';
import { StructuredLogger } from '../../../../common/logging/structured-logger.service';
import {
TbankAccessService,
TbankClient,
tbankSourceNeedsRefresh,
tbankSourceTtlMs,
type TbankBond,
type TbankShare,
} from '../../../../integrations/tbank/tbank.api';
import type { TbankLastPrice } from '../../../../integrations/tbank/tbank-market.types';
import type { TbankSyncAttempt } from '../../../../integrations/tbank/tbank-access.service';
import { investmentResourceMeta } from '../../investment-resource.mapper';
import {
CatalogDiscoveryRepository,
catalogScope,
type CatalogSource,
} from '../infrastructure/catalog-discovery.repository';
import { catalogSearchLimit } from './catalog-selection';
type Producer = {
input: CatalogSource;
attempt: TbankSyncAttempt;
observers: { attempt: TbankSyncAttempt; input: CatalogSource }[];
};
@Injectable()
export class CatalogEnrichmentService {
private readonly runId = randomUUID();
private readonly active = new Set<string>();
private readonly producers = new Map<string, Producer>();
private readonly queue: Producer[] = [];
private reservations: Promise<unknown> = Promise.resolve();
private running = false;
private readonly completions = new Map<
string,
{ workId: string; done: Promise<void>; resolve: () => void }
>();
async ensureDetail(
userId: string,
revision: string,
instrumentType: 'share' | 'bond',
uid: string,
) {
const ids = await this.requestMany(
userId,
revision,
instrumentType,
[uid],
false,
['detail'],
);
await Promise.all(
[...this.completions.values()]
.filter((entry) => ids.includes(entry.workId))
.map((entry) => entry.done),
);
const rows = await this.repository.readSources([
{ instrumentType, uid, kind: 'detail' },
]);
if (!rows.length) throw new Error('Instrument dependency missing');
return (rows[0].payload as CatalogPayload).instrument ?? null;
}
constructor(
private readonly repository: CatalogDiscoveryRepository,
private readonly access: TbankAccessService,
private readonly tbank: TbankClient,
private readonly logger: StructuredLogger,
private readonly moex: MoexClient,
private readonly ratings: NcrRatingsClient,
private readonly candles: CandleRepository,
) {}
async readMany(
userId: string,
instrumentType: 'share' | 'bond',
uids: string[],
observe: boolean,
kinds: CatalogSource['kind'][] = ['last-price', 'detail'],
range?: {
from: string;
to: string;
ticker?: string;
tickers?: Record<string, string>;
},
) {
const { revision } = await this.access.local(userId);
const inputs = sourceInputs(instrumentType, uids, kinds, range);
const rows = inputs.length ? await this.repository.readSources(inputs) : [];
if (!observe)
await this.requestMany(
userId,
revision,
instrumentType,
uids,
false,
kinds,
range,
);
const work = inputs.length
? await this.repository.workMany(userId, revision, inputs)
: [];
const byScope = new Map(work.map((row) => [row.scopeKey, row]));
return inputs.map((input) => {
const row = rows.find(
(entry) =>
entry.instrumentUid === input.uid && entry.kind === input.kind,
);
const payload = row?.payload as CatalogPayload | undefined;
return {
input,
version: row
? {
instrumentUid: row.instrumentUid,
source: row.source,
kind: row.kind,
rangeKey: row.rangeKey,
dataRevision: row.dataRevision,
fetchedAt: row.fetchedAt,
}
: null,
payload,
instrument: payload?.instrument ?? null,
price: payload?.price ?? null,
meta: investmentResourceMeta({
resourceKey: catalogScope(input),
dataRevision: String(row?.dataRevision ?? 0),
availability: row ? 'complete' : 'missing',
fetchedAt: row?.fetchedAt ?? null,
freshForMs: tbankSourceTtlMs[sourcePolicy(input.kind)],
work: byScope.get(catalogScope(input)) ?? null,
}),
};
});
}
requestMany(
userId: string,
revision: string,
instrumentType: 'share' | 'bond',
uids: string[],
force = false,
kinds: CatalogSource['kind'][] = ['last-price', 'detail'],
range?: {
from: string;
to: string;
ticker?: string;
tickers?: Record<string, string>;
},
) {
const inputs = sourceInputs(instrumentType, uids, kinds, range);
const reservation = this.reservations.then(async () => {
const rows = inputs.length
? await this.repository.readSources(inputs)
: [];
const ids: string[] = [];
for (const input of inputs) {
const row = rows.find(
(entry) =>
entry.instrumentUid === input.uid && entry.kind === input.kind,
);
if (
!tbankSourceNeedsRefresh(
sourcePolicy(input.kind),
row?.fetchedAt ?? null,
new Date(),
force,
)
)
continue;
const reserved = await this.repository.reserve(
{ userId, revision },
input,
this.runId,
this.active,
);
if (!reserved) continue;
const { id, attemptId } = reserved.work;
ids.push(id);
if (!reserved.launch) continue;
const attempt = { userId, revision, id, attemptId };
this.active.add(attemptId);
let resolve = () => {};
const done = new Promise<void>((finish) => {
resolve = finish;
});
this.completions.set(attemptId, { workId: id, done, resolve });
const key = producerKey(input);
const producer = this.producers.get(key);
if (producer) producer.observers.push({ attempt, input });
else {
const queued = { input, attempt, observers: [] };
this.producers.set(key, queued);
this.queue.push(queued);
}
}
if (this.queue.length && !this.running) {
this.running = true;
setImmediate(() => {
void this.drain();
});
}
return ids;
});
this.reservations = reservation.catch(() => undefined);
return reservation;
}
async section(
userId: string,
instrumentType: 'share' | 'bond',
uid: string,
section: 'quote' | 'company' | 'metrics',
observe = false,
) {
const kinds: CatalogSource['kind'][] =
section === 'company'
? ['detail', 'description', 'fundamentals']
: [
...(instrumentType === 'bond'
? ['bond-current' as const]
: ['detail' as const]),
'last-price',
'trading-status',
'comparison',
...(section === 'metrics'
? ['coupons' as const, 'yield' as const, 'rating' as const]
: []),
];
const sources = await this.readMany(
userId,
instrumentType,
[uid],
observe,
kinds,
);
const first = sources[0];
if (first.meta.availability === 'complete' && !first.instrument)
throw new NotFoundException();
const values = sources.reduce<CatalogPayload>(
(result, source) => ({ ...result, ...source.payload }),
{},
);
const dependencies = sources.map((source) => source.meta);
const complete = dependencies.every(
(dependency) => dependency.availability === 'complete',
);
const meta = investmentResourceMeta({
resourceKey: `${instrumentType}/${section}/${uid}`,
dataRevision: dependencies
.map((dependency) => dependency.dataRevision)
.join('/'),
availability: !first.instrument
? 'missing'
: complete
? 'complete'
: 'partial',
fetchedAt: first.meta.lastSuccessfulAt
? new Date(first.meta.lastSuccessfulAt)
: null,
freshForMs:
section === 'company'
? tbankSourceTtlMs.company
: tbankSourceTtlMs.quote,
work: null,
dependencies,
});
return {
values,
meta,
unavailable: sources
.filter(
(source) =>
source.meta.availability !== 'complete' ||
source.meta.sync.state === 'failed',
)
.map((source) => source.input.kind),
observedAt: meta.lastSuccessfulAt ?? new Date().toISOString(),
};
}
private async load(
input: CatalogSource,
token: string,
attempt: TbankSyncAttempt,
): Promise<CatalogPayload> {
if (input.kind === 'detail' || input.kind === 'bond-current') {
const instrument =
input.instrumentType === 'share'
? await this.tbank.getShare(token, input.uid)
: await this.tbank.getBond(token, input.uid);
if (instrument && instrument.uid !== input.uid)
throw new Error('Catalog UID mismatch');
if (instrument === undefined) throw new Error('Missing catalog response');
return { instrument: instrument ? publicInstrument(instrument) : null };
}
if (input.kind === 'trading-status')
return {
tradingStatus: await this.tbank.getTradingStatus(token, input.uid),
};
if (input.kind === 'comparison') {
const prices = await this.repository.readSources([
{ ...input, kind: 'last-price', rangeKey: undefined },
]);
if (!prices.length) throw new Error('Price dependency missing');
const price = (prices[0].payload as CatalogPayload).price;
const priceAt = timestamp(price?.time);
if (!priceAt) return { candle: null };
const now = new Date();
const range = {
from: utcDay(new Date(Date.parse(priceAt) - previousCloseLookbackMs)),
to: new Date(priceAt),
};
const saved = await this.candles.read(input.uid, range);
const gaps = candleGaps(range, saved.coverage, now);
for (const gap of gaps) {
const response = await this.tbank.getCandles(
token,
input.uid,
gap.from,
gap.to,
);
const rows = response.map((candle) => {
const time = timestamp(candle.time);
if (!time || !candle.close)
throw new Error('Invalid candle response');
return {
time,
close: formatDecimal(decimal(candle.close)),
isComplete: candle.isComplete === true,
};
});
if (!(await this.candles.stage(attempt, gap, rows, now)))
throw new Error('Cancelled candle comparison');
}
if (
gaps.length &&
!(await this.candles.publish(
attempt,
{ uid: input.uid, ranges: gaps },
false,
))
)
throw new Error('Cancelled candle comparison');
const values = (await this.candles.read(input.uid, range)).data.map(
(row) => {
const value = divide(
decimalFromString(row.close),
decimalFromString('1'),
quotationScale,
).value;
return {
time: {
seconds: String(Date.parse(row.time) / MILLISECONDS_PER_SECOND),
},
close: {
units: String(value / quotationUnits),
nano: Number(value % quotationUnits),
},
isComplete: row.isComplete,
};
},
);
return { candle: previousClose(values, priceAt) };
}
const details = await this.repository.readSources([
{ ...input, kind: 'detail', rangeKey: undefined },
...(input.instrumentType === 'bond'
? [{ ...input, kind: 'bond-current' as const, rangeKey: undefined }]
: []),
]);
const instrument = details
.map((row) => (row.payload as CatalogPayload).instrument)
.find(Boolean);
if (
!instrument &&
!['dividends', 'coupons', 'bond-events'].includes(input.kind)
)
throw new Error('Instrument dependency missing');
if (input.kind === 'description')
return {
description: instrument?.assetUid
? await this.tbank.getAssetDescription(token, instrument?.assetUid)
: null,
};
if (input.kind === 'fundamentals')
return {
fundamentals: instrument?.assetUid
? await this.tbank.getFundamentals(token, instrument?.assetUid)
: null,
};
if (input.kind === 'coupons' || input.kind === 'dividends') {
const year = Number(input.rangeKey);
const [from, to] = input.rangeKey?.startsWith('[')
? (JSON.parse(input.rangeKey) as [string, string])
: [
`${year - payoutHistoryYears}-01-01`,
`${year + payoutFutureYears}-01-01`,
];
return input.kind === 'coupons'
? {
coupons: await this.tbank.getBondCoupons(
token,
input.uid,
from,
to,
),
}
: {
dividends: await this.tbank.getDividends(
token,
input.uid,
from,
to,
),
};
}
if (input.kind === 'bond-events') {
const ticker = instrument?.ticker || input.ticker;
if (!ticker) throw new Error('Bond ticker missing');
return { bondEvents: await this.moex.bondEvents(ticker) };
}
if (input.kind === 'yield')
return {
effectiveYield:
instrument?.ticker && instrument.isin && instrument.classCode
? await this.moex.bondYield(
instrument.ticker,
instrument.isin,
instrument.classCode,
)
: null,
};
if (input.kind === 'rating')
return {
creditRating: instrument?.isin
? await this.ratings.rating(instrument.isin)
: null,
};
throw new Error('Unsupported catalog source');
}
private async drain() {
try {
while (this.queue.length) {
const producer = this.queue.shift();
if (!producer) break;
const batch = [producer];
// One batch RPC for queued prices sharing the same authorized producer.
if (producer.input.kind === 'last-price') {
for (
let i = 0;
i < this.queue.length && batch.length < catalogSearchLimit;
) {
const next = this.queue[i];
if (
next.input.kind === 'last-price' &&
next.attempt.userId === producer.attempt.userId &&
next.attempt.revision === producer.attempt.revision
) {
batch.push(next);
this.queue.splice(i, 1);
} else i++;
}
}
let prices: TbankLastPrice[] | undefined;
let payload: CatalogPayload | undefined;
let failed = false;
try {
const token = await this.access.tokenForAttempt(producer.attempt);
if (!token) failed = true;
else if (producer.input.kind === 'last-price')
prices = await this.tbank.getLastPrices(
token,
batch.map((item) => item.input.uid),
);
else
payload = await this.load(producer.input, token, producer.attempt);
} catch (error) {
failed = true;
if (error instanceof Error && error.message === 'INVALID_TBANK_TOKEN')
await this.access.rejectMembershipToken(producer.attempt);
else
this.logger.event('warn', 'catalog-enrichment.failed', {
code: 'TBANK_UNAVAILABLE',
});
}
for (const item of batch) {
const price =
prices?.find((entry) => entry.instrumentUid === item.input.uid) ??
null;
const value =
item.input.kind === 'last-price'
? {
price: price
? {
instrumentUid: item.input.uid,
price: price.price,
time: price.time,
}
: null,
}
: payload;
// Keep accepting observers while publication awaits SQLite.
item.observers.unshift({ attempt: item.attempt, input: item.input });
for (const observer of item.observers) {
const success =
!failed &&
value !== undefined &&
(await this.repository.publishSource(
observer.attempt,
observer.input,
value,
new Date(),
producer.attempt,
));
await this.repository.finish(observer.attempt, success);
this.active.delete(observer.attempt.attemptId);
this.completions.get(observer.attempt.attemptId)?.resolve();
this.completions.delete(observer.attempt.attemptId);
}
this.producers.delete(producerKey(item.input));
}
}
} catch {
for (const completion of this.completions.values()) completion.resolve();
this.completions.clear();
this.active.clear();
this.producers.clear();
this.queue.length = 0;
this.logger.event('error', 'catalog-enrichment.persistence-failed', {
code: 'TBANK_UNAVAILABLE',
});
} finally {
this.running = false;
}
}
}
function sourceInputs(
instrumentType: 'share' | 'bond',
uids: string[],
kinds: CatalogSource['kind'][],
range?: {
from: string;
to: string;
ticker?: string;
tickers?: Record<string, string>;
},
): CatalogSource[] {
return kinds.flatMap((kind) =>
[...new Set(uids)].map((uid) => ({
instrumentType,
uid,
kind,
...(kind === 'bond-events' && (range?.tickers?.[uid] || range?.ticker)
? { ticker: range?.tickers?.[uid] || range?.ticker }
: {}),
...(['coupons', 'dividends'].includes(kind)
? {
rangeKey: range
? JSON.stringify([range.from, range.to])
: String(new Date().getUTCFullYear()),
}
: {}),
})),
);
}
function publicInstrument(value: TbankShare | TbankBond) {
const bond = value as TbankBond;
return {
uid: value.uid,
figi: value.figi,
assetUid: value.assetUid,
name: value.name,
ticker: value.ticker,
isin: value.isin,
classCode: value.classCode,
realExchange: value.realExchange,
apiTradeAvailableFlag: value.apiTradeAvailableFlag,
currency: value.currency,
shareType: (value as TbankShare).shareType,
sector: value.sector,
lot: value.lot,
brand: value.brand ? { logoName: value.brand.logoName } : undefined,
first_1dayCandleDate: value.first_1dayCandleDate,
nominal: bond.nominal,
initialNominal: bond.initialNominal,
aciValue: bond.aciValue,
maturityDate: bond.maturityDate,
callDate: bond.callDate,
couponQuantityPerYear: bond.couponQuantityPerYear,
floatingCouponFlag: bond.floatingCouponFlag,
perpetualFlag: bond.perpetualFlag,
amortizationFlag: bond.amortizationFlag,
subordinatedFlag: bond.subordinatedFlag,
bondType: bond.bondType,
riskLevel: bond.riskLevel,
};
}
const previousCloseLookbackDays = 20;
const previousCloseLookbackMs =
previousCloseLookbackDays * MILLISECONDS_PER_DAY;
const payoutHistoryYears = 5;
const payoutFutureYears = 2;
export type CatalogPayload = {
instrument?: TbankShare | TbankBond | null;
price?: TbankLastPrice | null;
candle?: TbankCandle | null;
tradingStatus?: string | null;
description?: string | null;
fundamentals?: TbankFundamentals | null;
coupons?: TbankCoupon[];
dividends?: TbankDividend[];
bondEvents?: MoexBondEvent[];
effectiveYield?: Awaited<ReturnType<MoexClient['bondYield']>>;
creditRating?: Awaited<ReturnType<NcrRatingsClient['rating']>>;
};
function sourcePolicy(kind: CatalogSource['kind']) {
if (
[
'last-price',
'bond-current',
'trading-status',
'comparison',
'yield',
].includes(kind)
)
return 'quote';
if (kind === 'description' || kind === 'fundamentals') return 'company';
if (['coupons', 'dividends', 'bond-events'].includes(kind)) return 'calendar';
return 'metadata';
}
function producerKey(input: CatalogSource) {
return input.instrumentType === 'bond' &&
(input.kind === 'detail' || input.kind === 'bond-current')
? `bond-metadata:${input.uid}`
: catalogScope(input);
}
const quotationScale = 9;
const quotationUnits = 1_000_000_000n;