import { Injectable, NotFoundException } from '@nestjs/common'; import { randomUUID } from 'node:crypto'; import { decimalFromString, divide, } from '../../../../common/decimal'; import { StructuredLogger } from '../../../../common/logging/structured-logger.service'; import { MILLISECONDS_PER_DAY, MILLISECONDS_PER_SECOND, } from '../../../../common/time'; import { type MoexBondEvent, MoexClient } from '../../../../integrations/moex/moex.api'; import { NcrRatingsClient } from '../../../../integrations/ncr/ncr.api'; import { isTbankErrorCode, TbankErrorCode } from '../../../../integrations/tbank/tbank-error-code'; import { TbankAccessService, type TbankBond, type TbankCandle, TbankClient, type TbankCoupon, type TbankDividend, type TbankFundamentals, type TbankLastPrice, type TbankShare, tbankSourceNeedsRefresh, tbankSourceTtlMs, type TbankSyncAttempt, } from '../../../../integrations/tbank/tbank.api'; import { investmentResourceMeta } from '../../investment-resource.mapper'; import { candleGaps, utcDay } from '../domain/candle-coverage'; import { CandleRepository } from '../infrastructure/candle.repository'; import { CatalogDiscoveryRepository, catalogScope, type CatalogSource, } from '../infrastructure/catalog-discovery.repository'; import { toSavedCandle } from '../infrastructure/tbank-candle.mapper'; import { catalogSearchLimit } from './catalog-selection'; import { previousClose, timestamp } from './instrument-market'; const MISSING_DATA_REVISION = 0; const FIRST_QUEUE_INDEX = 0; const DEQUEUED_ITEM_COUNT = 1; type Producer = { attempt: TbankSyncAttempt; input: CatalogSource; observers: { attempt: TbankSyncAttempt; input: CatalogSource }[]; }; @Injectable() export class CatalogEnrichmentService { private readonly runId = randomUUID(); private readonly active = new Set(); private readonly producers = new Map(); private readonly queue: Producer[] = []; private reservations: Promise = Promise.resolve(); private running = false; private readonly completions = new Map< string, { done: Promise; resolve: () => void; workId: string } >(); async ensureDetail( userId: string, revision: string, instrumentType: 'bond' | 'share', uid: string, ) { const ids = await this.requestMany( userId, revision, instrumentType, [uid], false, ['detail'], ); await Promise.all( [...this.completions.values()] .filter((entry) => ids.includes(entry.workId)) .map((entry) => entry.done), ); const rows = await this.repository.readSources([ { instrumentType, uid, kind: 'detail' }, ]); if (!rows.length) throw new Error('Instrument dependency missing'); return (rows[0].payload as CatalogPayload).instrument ?? null; } constructor( private readonly repository: CatalogDiscoveryRepository, private readonly access: TbankAccessService, private readonly tbank: TbankClient, private readonly logger: StructuredLogger, private readonly moex: MoexClient, private readonly ratings: NcrRatingsClient, private readonly candles: CandleRepository, ) {} async readMany( userId: string, instrumentType: 'bond' | 'share', uids: string[], observe: boolean, kinds: CatalogSource['kind'][] = ['last-price', 'detail'], range?: { from: string; ticker?: string; tickers?: Record; to: string; }, ) { const { revision } = await this.access.local(userId); const inputs = sourceInputs(instrumentType, uids, kinds, range); const rows = inputs.length ? await this.repository.readSources(inputs) : []; if (!observe) { await this.requestMany( userId, revision, instrumentType, uids, false, kinds, range, ); } const work = inputs.length ? await this.repository.workMany(userId, revision, inputs) : []; const byScope = new Map(work.map((row) => [row.scopeKey, row])); return inputs.map((input) => { const row = rows.find( (entry) => entry.instrumentUid === input.uid && entry.kind === input.kind, ); const payload = row?.payload as CatalogPayload | undefined; return { input, version: row ? { instrumentUid: row.instrumentUid, source: row.source, kind: row.kind, rangeKey: row.rangeKey, dataRevision: row.dataRevision, fetchedAt: row.fetchedAt, } : null, payload, instrument: payload?.instrument ?? null, price: payload?.price ?? null, meta: investmentResourceMeta({ resourceKey: catalogScope(input), dataRevision: String(row?.dataRevision ?? MISSING_DATA_REVISION), availability: row ? 'complete' : 'missing', fetchedAt: row?.fetchedAt ?? null, freshForMs: tbankSourceTtlMs[sourcePolicy(input.kind)], work: byScope.get(catalogScope(input)) ?? null, }), }; }); } requestMany( userId: string, revision: string, instrumentType: 'bond' | 'share', uids: string[], force = false, kinds: CatalogSource['kind'][] = ['last-price', 'detail'], range?: { from: string; ticker?: string; tickers?: Record; to: string; }, ) { const inputs = sourceInputs(instrumentType, uids, kinds, range); const reservation = this.reservations.then(async () => { const rows = inputs.length ? await this.repository.readSources(inputs) : []; const ids: string[] = []; for (const input of inputs) { const row = rows.find( (entry) => entry.instrumentUid === input.uid && entry.kind === input.kind, ); if ( !tbankSourceNeedsRefresh( sourcePolicy(input.kind), row?.fetchedAt ?? null, new Date(), force, ) ) { continue; } const reserved = await this.repository.reserve( { userId, revision }, input, this.runId, this.active, ); if (!reserved) continue; const { id, attemptId } = reserved.work; ids.push(id); if (!reserved.launch) continue; const attempt = { userId, revision, id, attemptId }; this.active.add(attemptId); let resolve = () => {}; const done = new Promise((finish) => { resolve = finish; }); this.completions.set(attemptId, { workId: id, done, resolve }); const key = producerKey(input); const producer = this.producers.get(key); if (producer) producer.observers.push({ attempt, input }); else { const queued = { input, attempt, observers: [] }; this.producers.set(key, queued); this.queue.push(queued); } } if (this.queue.length && !this.running) { this.running = true; setImmediate(() => { void this.drain(); }); } return ids; }); this.reservations = reservation.catch(() => undefined); return reservation; } async section( userId: string, instrumentType: 'bond' | 'share', uid: string, section: 'company' | 'metrics' | 'quote', observe = false, ) { const kinds: CatalogSource['kind'][] = section === 'company' ? ['detail', 'description', 'fundamentals'] : [ ...(instrumentType === 'bond' ? ['bond-current' as const] : ['detail' as const]), 'last-price', 'trading-status', 'comparison', ...(section === 'metrics' ? ['coupons' as const, 'yield' as const, 'rating' as const] : []), ]; const sources = await this.readMany( userId, instrumentType, [uid], observe, kinds, ); const first = sources[0]; if (first.meta.availability === 'complete' && !first.instrument) { throw new NotFoundException(); } const values = sources.reduce( (result, source) => ({ ...result, ...source.payload }), {}, ); const dependencies = sources.map((source) => source.meta); const complete = dependencies.every( (dependency) => dependency.availability === 'complete', ); const meta = investmentResourceMeta({ resourceKey: `${instrumentType}/${section}/${uid}`, dataRevision: dependencies .map((dependency) => dependency.dataRevision) .join('/'), availability: !first.instrument ? 'missing' : complete ? 'complete' : 'partial', fetchedAt: first.meta.lastSuccessfulAt ? new Date(first.meta.lastSuccessfulAt) : null, freshForMs: section === 'company' ? tbankSourceTtlMs.company : tbankSourceTtlMs.quote, work: null, dependencies, }); return { values, meta, unavailable: sources .filter( (source) => source.meta.availability !== 'complete' || source.meta.sync.state === 'failed', ) .map((source) => source.input.kind), observedAt: meta.lastSuccessfulAt ?? new Date().toISOString(), }; } private async load( input: CatalogSource, token: string, attempt: TbankSyncAttempt, ): Promise { if (input.kind === 'detail' || input.kind === 'bond-current') { const instrument = input.instrumentType === 'share' ? await this.tbank.getShare(token, input.uid) : await this.tbank.getBond(token, input.uid); if (instrument && instrument.uid !== input.uid) { throw new Error('Catalog UID mismatch'); } // eslint-disable-next-line sonarjs/different-types-comparison -- Defensive validation rejects missing upstream payloads instead of caching them as absence. if (instrument === undefined) throw new Error('Missing catalog response'); return { instrument: instrument ? publicInstrument(instrument) : null }; } if (input.kind === 'trading-status') { return { tradingStatus: await this.tbank.getTradingStatus(token, input.uid), }; } if (input.kind === 'comparison') { const prices = await this.repository.readSources([ { ...input, kind: 'last-price', rangeKey: undefined }, ]); if (!prices.length) throw new Error('Price dependency missing'); const price = (prices[0].payload as CatalogPayload).price; const priceAt = timestamp(price?.time); if (!priceAt) return { candle: null }; const now = new Date(); const range = { from: utcDay(new Date(Date.parse(priceAt) - previousCloseLookbackMs)), to: new Date(priceAt), }; const saved = await this.candles.read(input.uid, range); const gaps = candleGaps(range, saved.coverage, now); for (const gap of gaps) { const response = await this.tbank.getCandles( token, input.uid, gap.from, gap.to, ); const rows = response.map(toSavedCandle); if (!(await this.candles.stage(attempt, gap, rows, now))) { throw new Error('Cancelled candle comparison'); } } if ( gaps.length && !(await this.candles.publish( attempt, { uid: input.uid, ranges: gaps }, false, )) ) { throw new Error('Cancelled candle comparison'); } const values = (await this.candles.read(input.uid, range)).data.map( (row) => { const value = divide( decimalFromString(row.close), decimalFromString('1'), quotationScale, ).value; return { time: { seconds: String(Date.parse(row.time) / MILLISECONDS_PER_SECOND), }, close: { units: String(value / quotationUnits), nano: Number(value % quotationUnits), }, isComplete: row.isComplete, }; }, ); return { candle: previousClose(values, priceAt) }; } const details = await this.repository.readSources([ { ...input, kind: 'detail', rangeKey: undefined }, ...(input.instrumentType === 'bond' ? [{ ...input, kind: 'bond-current' as const, rangeKey: undefined }] : []), ]); const instrument = details .map((row) => (row.payload as CatalogPayload).instrument) .find(Boolean); if ( !instrument && !['bond-events', 'coupons', 'dividends'].includes(input.kind) ) { throw new Error('Instrument dependency missing'); } if (input.kind === 'description') { return { description: instrument?.assetUid ? await this.tbank.getAssetDescription(token, instrument?.assetUid) : null, }; } if (input.kind === 'fundamentals') { return { fundamentals: instrument?.assetUid ? await this.tbank.getFundamentals(token, instrument?.assetUid) : null, }; } if (input.kind === 'coupons' || input.kind === 'dividends') { const year = Number(input.rangeKey); const [from, to] = input.rangeKey?.startsWith('[') ? (JSON.parse(input.rangeKey) as [string, string]) : [ `${year - payoutHistoryYears}-01-01`, `${year + payoutFutureYears}-01-01`, ]; return input.kind === 'coupons' ? { coupons: await this.tbank.getBondCoupons( token, input.uid, from, to, ), } : { dividends: await this.tbank.getDividends( token, input.uid, from, to, ), }; } if (input.kind === 'bond-events') { const ticker = instrument?.ticker || input.ticker; if (!ticker) throw new Error('Bond ticker missing'); return { bondEvents: await this.moex.bondEvents(ticker) }; } if (input.kind === 'yield') { return { effectiveYield: instrument?.ticker && instrument.isin && instrument.classCode ? await this.moex.bondYield( instrument.ticker, instrument.isin, instrument.classCode, ) : null, }; } if (input.kind === 'rating') { return { creditRating: instrument?.isin ? await this.ratings.rating(instrument.isin) : null, }; } throw new Error('Unsupported catalog source'); } private async drain() { try { while (this.queue.length) { const producer = this.queue.shift(); if (!producer) break; const batch = [producer]; // One batch RPC for queued prices sharing the same authorized producer. if (producer.input.kind === 'last-price') { for ( let i = FIRST_QUEUE_INDEX; i < this.queue.length && batch.length < catalogSearchLimit; ) { // eslint-disable-next-line security/detect-object-injection -- Numeric queue indexes and validated instrument UIDs select internal data only. const next = this.queue[i]; if ( next.input.kind === 'last-price' && next.attempt.userId === producer.attempt.userId && next.attempt.revision === producer.attempt.revision ) { batch.push(next); this.queue.splice(i, DEQUEUED_ITEM_COUNT); } else i++; } } let prices: TbankLastPrice[] | undefined; let payload: CatalogPayload | undefined; let failed = false; try { const token = await this.access.tokenForAttempt(producer.attempt); if (!token) failed = true; else if (producer.input.kind === 'last-price') { prices = await this.tbank.getLastPrices( token, batch.map((item) => item.input.uid), ); } else { payload = await this.load(producer.input, token, producer.attempt); } } catch (error) { failed = true; if (error instanceof Error && isTbankErrorCode(error.message, TbankErrorCode.InvalidToken)) { await this.access.rejectMembershipToken(producer.attempt); } else { this.logger.event('warn', 'catalog-enrichment.failed', { code: TbankErrorCode.Unavailable, }); } } for (const item of batch) { const price = prices?.find((entry) => entry.instrumentUid === item.input.uid) ?? null; const value = item.input.kind === 'last-price' ? { price: price ? { instrumentUid: item.input.uid, price: price.price, time: price.time, } : null, } : payload; // Keep accepting observers while publication awaits SQLite. item.observers.unshift({ attempt: item.attempt, input: item.input }); for (const observer of item.observers) { const success = !failed && value !== undefined && (await this.repository.publishSource( observer.attempt, observer.input, value, new Date(), producer.attempt, )); await this.repository.finish(observer.attempt, success); this.active.delete(observer.attempt.attemptId); this.completions.get(observer.attempt.attemptId)?.resolve(); this.completions.delete(observer.attempt.attemptId); } this.producers.delete(producerKey(item.input)); } } } catch { for (const completion of this.completions.values()) completion.resolve(); this.completions.clear(); this.active.clear(); this.producers.clear(); this.queue.length = 0; this.logger.event('error', 'catalog-enrichment.persistence-failed', { code: TbankErrorCode.Unavailable, }); } finally { this.running = false; } } } function sourceInputs( instrumentType: 'bond' | 'share', uids: string[], kinds: CatalogSource['kind'][], range?: { from: string; ticker?: string; tickers?: Record; to: string; }, ): CatalogSource[] { return kinds.flatMap((kind) => [...new Set(uids)].map((uid) => ({ instrumentType, uid, kind, // eslint-disable-next-line security/detect-object-injection -- Numeric queue indexes and validated instrument UIDs select internal data only. ...(kind === 'bond-events' && (range?.tickers?.[uid] || range?.ticker) // eslint-disable-next-line security/detect-object-injection -- Numeric queue indexes and validated instrument UIDs select internal data only. ? { ticker: range?.tickers?.[uid] || range?.ticker } : {}), ...(['coupons', 'dividends'].includes(kind) ? { rangeKey: range ? JSON.stringify([range.from, range.to]) : String(new Date().getUTCFullYear()), } : {}), })), ); } function publicInstrument(value: TbankBond | TbankShare) { const bond = value as TbankBond; return { uid: value.uid, figi: value.figi, assetUid: value.assetUid, name: value.name, ticker: value.ticker, isin: value.isin, classCode: value.classCode, realExchange: value.realExchange, apiTradeAvailableFlag: value.apiTradeAvailableFlag, currency: value.currency, shareType: (value as TbankShare).shareType, sector: value.sector, lot: value.lot, brand: value.brand ? { logoName: value.brand.logoName } : undefined, first_1dayCandleDate: value.first_1dayCandleDate, nominal: bond.nominal, initialNominal: bond.initialNominal, aciValue: bond.aciValue, maturityDate: bond.maturityDate, callDate: bond.callDate, couponQuantityPerYear: bond.couponQuantityPerYear, floatingCouponFlag: bond.floatingCouponFlag, perpetualFlag: bond.perpetualFlag, amortizationFlag: bond.amortizationFlag, subordinatedFlag: bond.subordinatedFlag, bondType: bond.bondType, riskLevel: bond.riskLevel, }; } const previousCloseLookbackDays = 20; const previousCloseLookbackMs = previousCloseLookbackDays * MILLISECONDS_PER_DAY; const payoutHistoryYears = 5; const payoutFutureYears = 2; export type CatalogPayload = { bondEvents?: MoexBondEvent[]; candle?: null | TbankCandle; coupons?: TbankCoupon[]; creditRating?: Awaited>; description?: null | string; dividends?: TbankDividend[]; effectiveYield?: Awaited>; fundamentals?: null | TbankFundamentals; instrument?: null | TbankBond | TbankShare; price?: null | TbankLastPrice; tradingStatus?: null | string; }; function sourcePolicy(kind: CatalogSource['kind']) { if ( [ 'bond-current', 'comparison', 'last-price', 'trading-status', 'yield', ].includes(kind) ) { return 'quote'; } if (kind === 'description' || kind === 'fundamentals') return 'company'; if (['bond-events', 'coupons', 'dividends'].includes(kind)) return 'calendar'; return 'metadata'; } function producerKey(input: CatalogSource) { return input.instrumentType === 'bond' && (input.kind === 'detail' || input.kind === 'bond-current') ? `bond-metadata:${input.uid}` : catalogScope(input); } const quotationScale = 9; const quotationUnits = 1_000_000_000n;