feat!: persist T-Bank data and synchronize reads
Backend security / security (push) Successful in 8s
Backend verify / verify (push) Failing after 10m45s
Conventional commits / commits (pull_request) Successful in 29s
Backend security / security (pull_request) Successful in 8s
Backend verify / verify (pull_request) Failing after 10m37s
Backend security / security (push) Successful in 8s
Backend verify / verify (push) Failing after 10m45s
Conventional commits / commits (pull_request) Successful in 29s
Backend security / security (pull_request) Successful in 8s
Backend verify / verify (pull_request) Failing after 10m37s
Store account membership, portfolio, operations, forecasts and market sources in SQLite. Coalesce scoped refreshes, fence connection revisions, preserve complete financial snapshots and stabilize operation period state. BREAKING CHANGE: investment reads return data/meta resource envelopes and locally persisted snapshots; consumers must use the matching OpenAPI contract and explicit synchronization endpoints.
This commit is contained in:
@@ -0,0 +1,481 @@
|
||||
import { planInvestmentScreen } from '../../investment-sync-plan';
|
||||
import { StructuredLogger } from '../../../../common/logging/structured-logger.service';
|
||||
import {
|
||||
Injectable,
|
||||
NotFoundException,
|
||||
ServiceUnavailableException,
|
||||
} from '@nestjs/common';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { MILLISECONDS_PER_DAY } from '../../../../common/time';
|
||||
import { formatDecimal } from '../../../../common/decimal';
|
||||
import { decimal } from '../../../../integrations/tbank/decimal';
|
||||
import {
|
||||
TbankAccessService,
|
||||
TbankClient,
|
||||
type TbankShare,
|
||||
} from '../../../../integrations/tbank/tbank.api';
|
||||
import type { TbankSyncAttempt } from '../../../../integrations/tbank/tbank-access.service';
|
||||
import { investmentResourceMeta } from '../../investment-resource.mapper';
|
||||
import { CatalogEnrichmentService } from './catalog-enrichment.service';
|
||||
import { CandleRepository } from '../infrastructure/candle.repository';
|
||||
import {
|
||||
candleGaps,
|
||||
candleWindows,
|
||||
utcDay,
|
||||
type CandleRange,
|
||||
} from '../domain/candle-coverage';
|
||||
import { timestamp } from './instrument-market';
|
||||
|
||||
type CandleInput = {
|
||||
uid: string;
|
||||
instrumentType: 'share' | 'bond';
|
||||
period: 'month' | 'halfYear' | 'year' | 'all';
|
||||
range?: CandleRange;
|
||||
};
|
||||
type Producer = {
|
||||
input: CandleInput;
|
||||
attempt: TbankSyncAttempt;
|
||||
now: Date;
|
||||
force: boolean;
|
||||
priority: number;
|
||||
observers: TbankSyncAttempt[];
|
||||
ranges?: CandleRange[];
|
||||
nextWindow?: number;
|
||||
};
|
||||
const months = { month: 1, halfYear: 6, year: 12 };
|
||||
const tailFreshForMs = 60_000;
|
||||
const prefetchReadBatchSize = 100;
|
||||
function scope(input: CandleInput) {
|
||||
return `candles:${input.uid}:${input.period}${input.range ? `:${input.range.from.toISOString()}/${input.range.to.toISOString()}` : ''}`;
|
||||
}
|
||||
function rangeFor(
|
||||
input: CandleInput,
|
||||
instrument: TbankShare,
|
||||
now: Date,
|
||||
): CandleRange {
|
||||
if (input.range) return input.range;
|
||||
const from = utcDay(now);
|
||||
const history = timestamp(instrument.first_1dayCandleDate);
|
||||
if (input.period === 'all') {
|
||||
if (!history)
|
||||
throw new ServiceUnavailableException({
|
||||
code:
|
||||
input.instrumentType === 'share'
|
||||
? 'STOCK_HISTORY_START_UNAVAILABLE'
|
||||
: 'BOND_HISTORY_START_UNAVAILABLE',
|
||||
});
|
||||
from.setTime(utcDay(new Date(history)).valueOf());
|
||||
} else {
|
||||
from.setUTCMonth(from.getUTCMonth() - months[input.period]);
|
||||
if (history && Date.parse(history) > from.valueOf())
|
||||
from.setTime(utcDay(new Date(history)).valueOf());
|
||||
}
|
||||
return { from, to: new Date(utcDay(now).valueOf() + MILLISECONDS_PER_DAY) };
|
||||
}
|
||||
@Injectable()
|
||||
export class CandleService {
|
||||
private readonly runId = randomUUID();
|
||||
private readonly active = new Set<string>();
|
||||
private readonly producers = new Map<string, Producer>();
|
||||
private readonly queue: Producer[] = [];
|
||||
private reservations: Promise<unknown> = Promise.resolve();
|
||||
private running = false;
|
||||
private readonly prefetches = new Map<string, Promise<void>>();
|
||||
constructor(
|
||||
private readonly repository: CandleRepository,
|
||||
private readonly sources: CatalogEnrichmentService,
|
||||
private readonly access: TbankAccessService,
|
||||
private readonly tbank: TbankClient,
|
||||
private readonly logger: StructuredLogger,
|
||||
) {}
|
||||
|
||||
async read(userId: string, input: CandleInput, observe = false) {
|
||||
const { revision } = await this.access.local(userId);
|
||||
const [detail] = await this.sources.readMany(
|
||||
userId,
|
||||
input.instrumentType,
|
||||
[input.uid],
|
||||
true,
|
||||
['detail'],
|
||||
);
|
||||
if (detail.meta.availability === 'complete' && !detail.instrument)
|
||||
throw new NotFoundException();
|
||||
const now = new Date();
|
||||
const range = detail.instrument
|
||||
? rangeFor(input, detail.instrument, now)
|
||||
: null;
|
||||
const saved = range ? await this.repository.read(input.uid, range) : null;
|
||||
const gaps = range && saved ? candleGaps(range, saved.coverage, now) : [];
|
||||
if (!observe && (!saved || gaps.length))
|
||||
await this.request(userId, revision, input, now);
|
||||
const work = await this.repository.work(userId, revision, scope(input));
|
||||
const fetchedAt =
|
||||
saved?.coverage
|
||||
.map((row) => row.fetchedAt)
|
||||
.sort((a, b) => a.valueOf() - b.valueOf())[0] ?? null;
|
||||
const covered = saved?.coverage.length ? true : false;
|
||||
const completeCoverage = (saved?.coverage ?? [])
|
||||
.filter((row) => row.isComplete || row.from >= utcDay(now))
|
||||
.map((row) => ({ ...row, isComplete: true }));
|
||||
const missing = range ? candleGaps(range, completeCoverage, now) : [];
|
||||
const complete = Boolean(saved && range && !missing.length);
|
||||
const publicRange = (value: CandleRange) => ({
|
||||
from: value.from.toISOString(),
|
||||
to: value.to.toISOString(),
|
||||
});
|
||||
return {
|
||||
data:
|
||||
!saved || !covered
|
||||
? null
|
||||
: {
|
||||
candles: saved.data,
|
||||
currency:
|
||||
input.instrumentType === 'share'
|
||||
? detail.instrument?.currency?.toUpperCase() || null
|
||||
: null,
|
||||
historyFrom: timestamp(detail?.instrument?.first_1dayCandleDate),
|
||||
observedAt: fetchedAt?.toISOString() ?? now.toISOString(),
|
||||
},
|
||||
meta: investmentResourceMeta({
|
||||
resourceKey: scope(input),
|
||||
dataRevision: saved?.dataRevision ?? '0',
|
||||
availability: !covered ? 'missing' : complete ? 'complete' : 'partial',
|
||||
fetchedAt,
|
||||
freshForMs: Number.POSITIVE_INFINITY,
|
||||
work,
|
||||
coverage: {
|
||||
requested: range ? [publicRange(range)] : [],
|
||||
completeRanges: range
|
||||
? completeCoverage.flatMap((row) => {
|
||||
const from = new Date(
|
||||
Math.max(row.from.valueOf(), range.from.valueOf()),
|
||||
);
|
||||
const to = new Date(
|
||||
Math.min(row.to.valueOf(), range.to.valueOf()),
|
||||
);
|
||||
return from < to ? [publicRange({ from, to })] : [];
|
||||
})
|
||||
: [],
|
||||
missingRanges: missing.map(publicRange),
|
||||
asOf: fetchedAt?.toISOString() ?? null,
|
||||
historyBoundary: timestamp(detail?.instrument?.first_1dayCandleDate)
|
||||
? 'known'
|
||||
: 'unknown',
|
||||
historyState: complete
|
||||
? 'complete'
|
||||
: work?.state === 'failed'
|
||||
? 'failed'
|
||||
: work?.state === 'running'
|
||||
? 'loading'
|
||||
: covered
|
||||
? 'partial'
|
||||
: 'missing',
|
||||
},
|
||||
dependencies: [
|
||||
detail.meta,
|
||||
...(saved?.coverage ?? []).map((row) =>
|
||||
investmentResourceMeta({
|
||||
resourceKey: `${scope(input)}/${row.from.toISOString()}/${row.to.toISOString()}`,
|
||||
dataRevision: String(row.dataRevision),
|
||||
availability: 'complete',
|
||||
fetchedAt: row.fetchedAt,
|
||||
freshForMs: row.isComplete
|
||||
? Number.POSITIVE_INFINITY
|
||||
: tailFreshForMs,
|
||||
work: null,
|
||||
}),
|
||||
),
|
||||
],
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
prefetch(
|
||||
userId: string,
|
||||
revision: string,
|
||||
positions: { instrumentUid?: string | null; type: string }[],
|
||||
) {
|
||||
const key = JSON.stringify([
|
||||
userId,
|
||||
revision,
|
||||
positions
|
||||
.map((position) => [position.instrumentUid, position.type])
|
||||
.sort(),
|
||||
]);
|
||||
const existing = this.prefetches.get(key);
|
||||
if (existing) return existing;
|
||||
const work = this.prefetchPositions(userId, revision, positions).finally(
|
||||
() => this.prefetches.delete(key),
|
||||
);
|
||||
this.prefetches.set(key, work);
|
||||
return work;
|
||||
}
|
||||
|
||||
private async prefetchPositions(
|
||||
userId: string,
|
||||
revision: string,
|
||||
positions: { instrumentUid?: string | null; type: string }[],
|
||||
) {
|
||||
const now = new Date();
|
||||
const unique = new Map(
|
||||
positions
|
||||
.filter(
|
||||
(position) =>
|
||||
position.instrumentUid && ['STOCK', 'BOND'].includes(position.type),
|
||||
)
|
||||
.map((position) => [position.instrumentUid ?? '', position]),
|
||||
);
|
||||
const details: Awaited<ReturnType<CatalogEnrichmentService['readMany']>> =
|
||||
[];
|
||||
const coverage: Awaited<ReturnType<CandleRepository['coverage']>> = [];
|
||||
for (const instrumentType of ['share', 'bond'] as const) {
|
||||
const uids = [...unique]
|
||||
.filter(
|
||||
([, position]) =>
|
||||
position.type === (instrumentType === 'share' ? 'STOCK' : 'BOND'),
|
||||
)
|
||||
.map(([uid]) => uid);
|
||||
for (
|
||||
let offset = 0;
|
||||
offset < uids.length;
|
||||
offset += prefetchReadBatchSize
|
||||
) {
|
||||
const batch = uids.slice(offset, offset + prefetchReadBatchSize);
|
||||
details.push(
|
||||
...(await this.sources.readMany(userId, instrumentType, batch, true, [
|
||||
'detail',
|
||||
])),
|
||||
);
|
||||
coverage.push(...(await this.repository.coverage(batch)));
|
||||
}
|
||||
}
|
||||
const byUid = new Map(details.map((detail) => [detail.input.uid, detail]));
|
||||
for (const [uid, position] of unique) {
|
||||
const instrumentType: CandleInput['instrumentType'] =
|
||||
position.type === 'STOCK' ? 'share' : 'bond';
|
||||
const detail = byUid.get(uid);
|
||||
const history = timestamp(detail?.instrument?.first_1dayCandleDate);
|
||||
const plan = planInvestmentScreen(
|
||||
{ screen: 'account' },
|
||||
{
|
||||
membershipKnown: true,
|
||||
accounts: [],
|
||||
instruments: [
|
||||
{
|
||||
uid,
|
||||
type: position.type as 'STOCK' | 'BOND',
|
||||
historyFrom: history ? new Date(history) : null,
|
||||
},
|
||||
],
|
||||
},
|
||||
now,
|
||||
);
|
||||
if (
|
||||
!detail?.instrument ||
|
||||
plan.sources.some((source) => source.kind === 'candles')
|
||||
) {
|
||||
const input = { uid, instrumentType, period: 'all' as const };
|
||||
const range =
|
||||
detail?.instrument && history
|
||||
? rangeFor(input, detail.instrument, now)
|
||||
: null;
|
||||
if (
|
||||
!range ||
|
||||
candleGaps(
|
||||
range,
|
||||
coverage.filter((row) => row.instrumentUid === uid),
|
||||
now,
|
||||
).length
|
||||
)
|
||||
await this.request(userId, revision, input, now, false, 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
request(
|
||||
userId: string,
|
||||
revision: string,
|
||||
input: CandleInput,
|
||||
now = new Date(),
|
||||
force = false,
|
||||
priority = 1,
|
||||
) {
|
||||
const pending = this.reservations.then(async () => {
|
||||
const reserved = await this.repository.reserve(
|
||||
{ userId, revision },
|
||||
scope(input),
|
||||
{
|
||||
uid: input.uid,
|
||||
instrumentType: input.instrumentType,
|
||||
ranges: [],
|
||||
period: input.period,
|
||||
now: now.toISOString(),
|
||||
force,
|
||||
},
|
||||
this.runId,
|
||||
this.active,
|
||||
priority,
|
||||
);
|
||||
if (!reserved) return null;
|
||||
const attempt = {
|
||||
userId,
|
||||
revision,
|
||||
id: reserved.work.id,
|
||||
attemptId: reserved.work.attemptId,
|
||||
};
|
||||
if (reserved.launch) {
|
||||
this.active.add(attempt.attemptId);
|
||||
const producer = this.producers.get(scope(input));
|
||||
if (producer) producer.observers.push(attempt);
|
||||
else {
|
||||
const next = { input, attempt, now, force, priority, observers: [] };
|
||||
this.producers.set(scope(input), next);
|
||||
this.queue.push(next);
|
||||
}
|
||||
if (!this.running) {
|
||||
this.running = true;
|
||||
setImmediate(() => {
|
||||
void this.drain();
|
||||
});
|
||||
}
|
||||
}
|
||||
return attempt.id;
|
||||
});
|
||||
this.reservations = pending.catch(() => undefined);
|
||||
return pending;
|
||||
}
|
||||
|
||||
private async load(producer: Producer) {
|
||||
const { input, attempt, now } = producer;
|
||||
const instrument = await this.sources.ensureDetail(
|
||||
attempt.userId,
|
||||
attempt.revision,
|
||||
input.instrumentType,
|
||||
input.uid,
|
||||
);
|
||||
if (!instrument) throw new NotFoundException();
|
||||
const range = rangeFor(input, instrument, now);
|
||||
if (!producer.ranges) {
|
||||
const saved = await this.repository.read(input.uid, range);
|
||||
producer.ranges = (
|
||||
producer.force ? [range] : candleGaps(range, saved.coverage, now)
|
||||
)
|
||||
.flatMap(candleWindows)
|
||||
.reverse();
|
||||
producer.nextWindow = 0;
|
||||
}
|
||||
const ranges = producer.ranges;
|
||||
const index = producer.nextWindow ?? 0;
|
||||
let window = ranges[index];
|
||||
if (window && !producer.force) {
|
||||
const latest = await this.repository.read(input.uid, window);
|
||||
const gaps = candleGaps(window, latest.coverage, now)
|
||||
.flatMap(candleWindows)
|
||||
.reverse();
|
||||
ranges.splice(index, 1, ...gaps);
|
||||
window = ranges[index];
|
||||
}
|
||||
if (window) {
|
||||
const token = await this.access.tokenForAttempt(attempt);
|
||||
if (!token) return false;
|
||||
const candles = await this.tbank.getCandles(
|
||||
token,
|
||||
input.uid,
|
||||
window.from,
|
||||
new Date(Math.min(window.to.valueOf(), now.valueOf())),
|
||||
);
|
||||
const rows = candles.flatMap((candle) => {
|
||||
const time = timestamp(candle.time);
|
||||
if (!time || !candle.close) throw new Error('Invalid candle response');
|
||||
return [
|
||||
{
|
||||
time,
|
||||
close: formatDecimal(decimal(candle.close)),
|
||||
isComplete: candle.isComplete === true,
|
||||
},
|
||||
];
|
||||
});
|
||||
if (!(await this.repository.stage(attempt, window, rows, now)))
|
||||
return false;
|
||||
if (
|
||||
!producer.force &&
|
||||
!(await this.repository.publish(
|
||||
attempt,
|
||||
{ uid: input.uid, ranges: [window] },
|
||||
false,
|
||||
))
|
||||
)
|
||||
return false;
|
||||
producer.nextWindow = index + 1;
|
||||
if (producer.nextWindow < ranges.length) return null;
|
||||
}
|
||||
return this.repository.publish(attempt, {
|
||||
uid: input.uid,
|
||||
ranges: producer.force ? ranges : [],
|
||||
});
|
||||
}
|
||||
|
||||
private async drain() {
|
||||
try {
|
||||
while (this.queue.length) {
|
||||
this.queue.sort((a, b) => a.priority - b.priority);
|
||||
const producer = this.queue.shift();
|
||||
if (!producer) break;
|
||||
let success = false;
|
||||
let code = 'TBANK_UNAVAILABLE';
|
||||
try {
|
||||
const result = await this.load(producer);
|
||||
if (result === null) {
|
||||
this.queue.push(producer);
|
||||
continue;
|
||||
}
|
||||
success = result;
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message === 'INVALID_TBANK_TOKEN')
|
||||
await this.access.rejectMembershipToken(producer.attempt);
|
||||
if (error instanceof ServiceUnavailableException) {
|
||||
const response = error.getResponse();
|
||||
if (
|
||||
typeof response === 'object' &&
|
||||
'code' in response &&
|
||||
typeof response.code === 'string'
|
||||
)
|
||||
code = response.code;
|
||||
}
|
||||
}
|
||||
if (!success) await this.repository.fail(producer.attempt, code);
|
||||
this.active.delete(producer.attempt.attemptId);
|
||||
for (const observer of producer.observers) {
|
||||
if (
|
||||
!success ||
|
||||
!(await this.repository.completeObserver(
|
||||
observer,
|
||||
producer.attempt,
|
||||
))
|
||||
)
|
||||
await this.repository.fail(observer, code);
|
||||
this.active.delete(observer.attemptId);
|
||||
}
|
||||
this.producers.delete(scope(producer.input));
|
||||
if (success && !producer.force && producer.input.period !== 'all')
|
||||
await this.request(
|
||||
producer.attempt.userId,
|
||||
producer.attempt.revision,
|
||||
{ ...producer.input, period: 'all' },
|
||||
producer.now,
|
||||
false,
|
||||
2,
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
this.logger.event('error', 'candles.persistence-failed', {
|
||||
code: 'TBANK_UNAVAILABLE',
|
||||
});
|
||||
} finally {
|
||||
this.active.clear();
|
||||
this.producers.clear();
|
||||
this.queue.length = 0;
|
||||
this.running = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,390 @@
|
||||
import { CatalogEnrichmentService } from './catalog-enrichment.service';
|
||||
import {
|
||||
Injectable,
|
||||
NotFoundException,
|
||||
ConflictException,
|
||||
} from '@nestjs/common';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { StructuredLogger } from '../../../../common/logging/structured-logger.service';
|
||||
import {
|
||||
TbankAccessService,
|
||||
TbankClient,
|
||||
tbankSourceNeedsRefresh,
|
||||
tbankSourceTtlMs,
|
||||
} from '../../../../integrations/tbank/tbank.api';
|
||||
import type { TbankSyncAttempt } from '../../../../integrations/tbank/tbank-access.service';
|
||||
import { investmentResourceMeta } from '../../investment-resource.mapper';
|
||||
import {
|
||||
CatalogDiscoveryRepository,
|
||||
catalogQuery,
|
||||
catalogScope,
|
||||
type CatalogQuery,
|
||||
} from '../infrastructure/catalog-discovery.repository';
|
||||
import {
|
||||
catalogSearchLimit,
|
||||
selectCatalogInstruments,
|
||||
} from './catalog-selection';
|
||||
|
||||
const tickerAmbiguityLimit = 2;
|
||||
|
||||
type Producer = {
|
||||
input: CatalogQuery;
|
||||
attempt: TbankSyncAttempt;
|
||||
observers: TbankSyncAttempt[];
|
||||
force: boolean;
|
||||
};
|
||||
@Injectable()
|
||||
export class CatalogDiscoveryService {
|
||||
private readonly runId = randomUUID();
|
||||
private readonly active = new Set<string>();
|
||||
private readonly producers = new Map<string, Producer>();
|
||||
private readonly queue: Producer[] = [];
|
||||
private reservations: Promise<unknown> = Promise.resolve();
|
||||
private running = false;
|
||||
private readonly completions = new Map<
|
||||
string,
|
||||
{ workId: string; done: Promise<void>; resolve: () => void }
|
||||
>();
|
||||
constructor(
|
||||
private readonly repository: CatalogDiscoveryRepository,
|
||||
private readonly access: TbankAccessService,
|
||||
private readonly tbank: TbankClient,
|
||||
private readonly logger: StructuredLogger,
|
||||
private readonly enrichment: CatalogEnrichmentService,
|
||||
) {}
|
||||
|
||||
async read(userId: string, input: CatalogQuery, observe = false) {
|
||||
input = catalogQuery(input);
|
||||
const connection = await this.access.local(userId);
|
||||
const saved = await this.repository.read(input);
|
||||
if (!observe) await this.request(userId, connection.revision, input);
|
||||
return {
|
||||
data: saved.data,
|
||||
meta: investmentResourceMeta({
|
||||
resourceKey: catalogScope(input),
|
||||
dataRevision: saved.dataRevision,
|
||||
availability: saved.data === null ? 'missing' : 'complete',
|
||||
fetchedAt: saved.fetchedAt,
|
||||
freshForMs: tbankSourceTtlMs.catalog,
|
||||
work: await this.repository.work(userId, connection.revision, input),
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
async search(userId: string, input: CatalogQuery, observe = false) {
|
||||
const discovery = await this.read(userId, input, observe);
|
||||
const sources = discovery.data
|
||||
? await this.enrichment.readMany(
|
||||
userId,
|
||||
input.instrumentType,
|
||||
discovery.data.map((item) => item.uid),
|
||||
observe,
|
||||
)
|
||||
: [];
|
||||
const dependencies = [
|
||||
discovery.meta,
|
||||
...sources.map((source) => source.meta),
|
||||
];
|
||||
return {
|
||||
data:
|
||||
discovery.data?.map((short) => ({
|
||||
...short,
|
||||
detail:
|
||||
sources.find(
|
||||
(source) =>
|
||||
source.input.uid === short.uid &&
|
||||
source.input.kind === 'detail',
|
||||
)?.instrument ?? null,
|
||||
price:
|
||||
sources.find(
|
||||
(source) =>
|
||||
source.input.uid === short.uid &&
|
||||
source.input.kind === 'last-price',
|
||||
)?.price ?? null,
|
||||
})) ?? null,
|
||||
meta: investmentResourceMeta({
|
||||
resourceKey: `search:${catalogScope(catalogQuery(input))}`,
|
||||
dataRevision: dependencies
|
||||
.map((dependency) => dependency.dataRevision)
|
||||
.join('/'),
|
||||
availability:
|
||||
discovery.data === null
|
||||
? 'missing'
|
||||
: sources.every((source) => source.meta.availability === 'complete')
|
||||
? 'complete'
|
||||
: 'partial',
|
||||
fetchedAt: discovery.meta.lastSuccessfulAt
|
||||
? new Date(discovery.meta.lastSuccessfulAt)
|
||||
: null,
|
||||
freshForMs: tbankSourceTtlMs.catalog,
|
||||
work: discovery.meta.sync.id
|
||||
? {
|
||||
id: discovery.meta.sync.id,
|
||||
state: discovery.meta.sync.state,
|
||||
errorCode: discovery.meta.sync.errorCode,
|
||||
}
|
||||
: null,
|
||||
dependencies,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
async detail(
|
||||
userId: string,
|
||||
instrumentType: 'share' | 'bond',
|
||||
uid: string,
|
||||
observe = false,
|
||||
) {
|
||||
const sources = await this.enrichment.readMany(
|
||||
userId,
|
||||
instrumentType,
|
||||
[uid],
|
||||
observe,
|
||||
['detail'],
|
||||
);
|
||||
const source = sources[0];
|
||||
if (source.meta.availability === 'complete' && !source.instrument)
|
||||
throw new NotFoundException();
|
||||
const instrument = source.instrument;
|
||||
let slug = uid;
|
||||
const dependencies = [source.meta];
|
||||
if (
|
||||
instrument?.ticker &&
|
||||
/^[A-Za-z0-9][A-Za-z0-9._@+-]{0,79}$/.test(instrument.ticker)
|
||||
) {
|
||||
const alias = await this.read(
|
||||
userId,
|
||||
{ instrumentType, query: instrument.ticker, exactTicker: true },
|
||||
observe,
|
||||
);
|
||||
if (alias.data?.length === 1 && alias.data[0].uid === uid)
|
||||
slug = instrument.ticker.toUpperCase();
|
||||
dependencies.push(alias.meta);
|
||||
}
|
||||
return {
|
||||
data: instrument ? { ...instrument, uid, slug } : null,
|
||||
meta: investmentResourceMeta({
|
||||
resourceKey: `${instrumentType}/detail/${uid}`,
|
||||
dataRevision: dependencies
|
||||
.map((dependency) => dependency.dataRevision)
|
||||
.join('/'),
|
||||
availability: source.meta.availability,
|
||||
fetchedAt: source.meta.lastSuccessfulAt
|
||||
? new Date(source.meta.lastSuccessfulAt)
|
||||
: null,
|
||||
freshForMs: tbankSourceTtlMs.metadata,
|
||||
work: source.meta.sync.id
|
||||
? {
|
||||
id: source.meta.sync.id,
|
||||
state: source.meta.sync.state,
|
||||
errorCode: source.meta.sync.errorCode,
|
||||
}
|
||||
: null,
|
||||
dependencies,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
async byTicker(
|
||||
userId: string,
|
||||
instrumentType: 'share' | 'bond',
|
||||
ticker: string,
|
||||
observe = false,
|
||||
) {
|
||||
const lookup = await this.read(
|
||||
userId,
|
||||
{ instrumentType, query: ticker, exactTicker: true },
|
||||
observe,
|
||||
);
|
||||
if (lookup.data?.length === 0) throw new NotFoundException();
|
||||
if (lookup.data && lookup.data.length > 1)
|
||||
throw new ConflictException({
|
||||
code:
|
||||
instrumentType === 'share'
|
||||
? 'STOCK_TICKER_AMBIGUOUS'
|
||||
: 'BOND_TICKER_AMBIGUOUS',
|
||||
});
|
||||
const result = lookup.data
|
||||
? await this.detail(userId, instrumentType, lookup.data[0].uid, observe)
|
||||
: null;
|
||||
const dependencies = [lookup.meta, ...(result ? [result.meta] : [])];
|
||||
return {
|
||||
data: result?.data
|
||||
? { ...result.data, slug: ticker.trim().toUpperCase() }
|
||||
: null,
|
||||
meta: investmentResourceMeta({
|
||||
resourceKey: `${instrumentType}/ticker/${ticker.trim().toUpperCase()}`,
|
||||
dataRevision: dependencies
|
||||
.map((dependency) => dependency.dataRevision)
|
||||
.join('/'),
|
||||
availability: result?.meta.availability ?? 'missing',
|
||||
fetchedAt: lookup.meta.lastSuccessfulAt
|
||||
? new Date(lookup.meta.lastSuccessfulAt)
|
||||
: null,
|
||||
freshForMs: tbankSourceTtlMs.metadata,
|
||||
work: lookup.meta.sync.id
|
||||
? {
|
||||
id: lookup.meta.sync.id,
|
||||
state: lookup.meta.sync.state,
|
||||
errorCode: lookup.meta.sync.errorCode,
|
||||
}
|
||||
: null,
|
||||
dependencies,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
request(
|
||||
userId: string,
|
||||
revision: string,
|
||||
query: CatalogQuery,
|
||||
force = false,
|
||||
) {
|
||||
const input = catalogQuery(query);
|
||||
const reserved = this.reservations.then(async () => {
|
||||
const saved = await this.repository.read(input);
|
||||
if (
|
||||
!tbankSourceNeedsRefresh('catalog', saved.fetchedAt, new Date(), force)
|
||||
)
|
||||
return null;
|
||||
const reservation = await this.repository.reserve(
|
||||
{ userId, revision },
|
||||
input,
|
||||
this.runId,
|
||||
this.active,
|
||||
);
|
||||
if (!reservation) return null;
|
||||
const { id, attemptId } = reservation.work;
|
||||
if (!reservation.launch) {
|
||||
const current = this.producers.get(catalogScope(input));
|
||||
if (current) current.force ||= force;
|
||||
return id;
|
||||
}
|
||||
const attempt = { userId, revision, id, attemptId };
|
||||
this.active.add(attemptId);
|
||||
let resolve = () => {};
|
||||
const done = new Promise<void>((finish) => {
|
||||
resolve = finish;
|
||||
});
|
||||
this.completions.set(attemptId, { workId: id, done, resolve });
|
||||
const key = catalogScope(input);
|
||||
const producer = this.producers.get(key);
|
||||
if (producer) {
|
||||
producer.observers.push(attempt);
|
||||
producer.force ||= force;
|
||||
} else {
|
||||
const queued = { input, attempt, observers: [], force };
|
||||
this.producers.set(key, queued);
|
||||
this.queue.push(queued);
|
||||
}
|
||||
if (!this.running) {
|
||||
this.running = true;
|
||||
setImmediate(() => {
|
||||
void this.drain();
|
||||
});
|
||||
}
|
||||
return id;
|
||||
});
|
||||
this.reservations = reserved.catch(() => undefined);
|
||||
return reserved;
|
||||
}
|
||||
|
||||
async refresh(userId: string, revision: string, query: CatalogQuery) {
|
||||
const id = await this.request(userId, revision, query, true);
|
||||
if (id)
|
||||
await Promise.all(
|
||||
[...this.completions.values()]
|
||||
.filter((entry) => entry.workId === id)
|
||||
.map((entry) => entry.done),
|
||||
);
|
||||
const saved = await this.repository.read(catalogQuery(query));
|
||||
const sources = await this.enrichment.readMany(
|
||||
userId,
|
||||
query.instrumentType,
|
||||
saved.data?.map((item) => item.uid) ?? [],
|
||||
true,
|
||||
);
|
||||
return [
|
||||
...(id ? [id] : []),
|
||||
...sources.flatMap((source) =>
|
||||
source.meta.sync.id ? [source.meta.sync.id] : [],
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
private async drain() {
|
||||
try {
|
||||
while (this.queue.length) {
|
||||
const producer = this.queue.shift();
|
||||
if (!producer) break;
|
||||
const { input, attempt } = producer;
|
||||
let success = false;
|
||||
try {
|
||||
const token = await this.access.tokenForAttempt(attempt);
|
||||
if (token) {
|
||||
const found =
|
||||
input.instrumentType === 'share'
|
||||
? await this.tbank.findShares(token, input.query)
|
||||
: await this.tbank.findBonds(token, input.query);
|
||||
success = await this.repository.publish(
|
||||
attempt,
|
||||
input,
|
||||
selectCatalogInstruments(
|
||||
input.exactTicker
|
||||
? found.filter(
|
||||
(item) =>
|
||||
item.ticker?.trim().toUpperCase() === input.query,
|
||||
)
|
||||
: found,
|
||||
input.query,
|
||||
input.instrumentType,
|
||||
).slice(
|
||||
0,
|
||||
input.exactTicker ? tickerAmbiguityLimit : catalogSearchLimit,
|
||||
),
|
||||
new Date(),
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message === 'INVALID_TBANK_TOKEN')
|
||||
await this.access.rejectMembershipToken(attempt);
|
||||
else
|
||||
this.logger.event('warn', 'catalog-discovery.failed', {
|
||||
code: 'TBANK_UNAVAILABLE',
|
||||
});
|
||||
}
|
||||
if (success) {
|
||||
const saved = await this.repository.read(input);
|
||||
for (const observer of [attempt, ...producer.observers])
|
||||
if (!input.exactTicker || saved.data?.length === 1)
|
||||
await this.enrichment.requestMany(
|
||||
observer.userId,
|
||||
observer.revision,
|
||||
input.instrumentType,
|
||||
saved.data?.map((item) => item.uid) ?? [],
|
||||
producer.force,
|
||||
input.exactTicker ? ['detail'] : ['last-price', 'detail'],
|
||||
);
|
||||
}
|
||||
this.producers.delete(catalogScope(input));
|
||||
for (const observer of [attempt, ...producer.observers]) {
|
||||
await this.repository.finish(observer, success);
|
||||
this.active.delete(observer.attemptId);
|
||||
this.completions.get(observer.attemptId)?.resolve();
|
||||
this.completions.delete(observer.attemptId);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
for (const completion of this.completions.values()) completion.resolve();
|
||||
this.completions.clear();
|
||||
this.active.clear();
|
||||
this.producers.clear();
|
||||
this.queue.length = 0;
|
||||
this.logger.event('error', 'catalog-discovery.persistence-failed', {
|
||||
code: 'TBANK_UNAVAILABLE',
|
||||
});
|
||||
} finally {
|
||||
this.running = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,665 @@
|
||||
import type { TbankDividend } from '../../../../integrations/tbank/tbank.api';
|
||||
import type { MoexBondEvent } from '../../../../integrations/moex/moex.api';
|
||||
import { CandleRepository } from '../infrastructure/candle.repository';
|
||||
import { candleGaps, utcDay } from '../domain/candle-coverage';
|
||||
import {
|
||||
decimalFromString,
|
||||
divide,
|
||||
formatDecimal,
|
||||
} from '../../../../common/decimal';
|
||||
import { decimal } from '../../../../integrations/tbank/decimal';
|
||||
import {
|
||||
MILLISECONDS_PER_DAY,
|
||||
MILLISECONDS_PER_SECOND,
|
||||
} from '../../../../common/time';
|
||||
import { MoexClient } from '../../../../integrations/moex/moex.api';
|
||||
import { NcrRatingsClient } from '../../../../integrations/ncr/ncr.api';
|
||||
import type {
|
||||
TbankCandle,
|
||||
TbankFundamentals,
|
||||
} from '../../../../integrations/tbank/tbank.api';
|
||||
import type { TbankCoupon } from '../../../../integrations/tbank/tbank-market.types';
|
||||
import { previousClose, timestamp } from './instrument-market';
|
||||
import { Injectable, NotFoundException } from '@nestjs/common';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { StructuredLogger } from '../../../../common/logging/structured-logger.service';
|
||||
import {
|
||||
TbankAccessService,
|
||||
TbankClient,
|
||||
tbankSourceNeedsRefresh,
|
||||
tbankSourceTtlMs,
|
||||
type TbankBond,
|
||||
type TbankShare,
|
||||
} from '../../../../integrations/tbank/tbank.api';
|
||||
import type { TbankLastPrice } from '../../../../integrations/tbank/tbank-market.types';
|
||||
import type { TbankSyncAttempt } from '../../../../integrations/tbank/tbank-access.service';
|
||||
import { investmentResourceMeta } from '../../investment-resource.mapper';
|
||||
import {
|
||||
CatalogDiscoveryRepository,
|
||||
catalogScope,
|
||||
type CatalogSource,
|
||||
} from '../infrastructure/catalog-discovery.repository';
|
||||
import { catalogSearchLimit } from './catalog-selection';
|
||||
|
||||
type Producer = {
|
||||
input: CatalogSource;
|
||||
attempt: TbankSyncAttempt;
|
||||
observers: { attempt: TbankSyncAttempt; input: CatalogSource }[];
|
||||
};
|
||||
@Injectable()
|
||||
export class CatalogEnrichmentService {
|
||||
private readonly runId = randomUUID();
|
||||
private readonly active = new Set<string>();
|
||||
private readonly producers = new Map<string, Producer>();
|
||||
private readonly queue: Producer[] = [];
|
||||
private reservations: Promise<unknown> = Promise.resolve();
|
||||
private running = false;
|
||||
private readonly completions = new Map<
|
||||
string,
|
||||
{ workId: string; done: Promise<void>; resolve: () => void }
|
||||
>();
|
||||
|
||||
async ensureDetail(
|
||||
userId: string,
|
||||
revision: string,
|
||||
instrumentType: 'share' | 'bond',
|
||||
uid: string,
|
||||
) {
|
||||
const ids = await this.requestMany(
|
||||
userId,
|
||||
revision,
|
||||
instrumentType,
|
||||
[uid],
|
||||
false,
|
||||
['detail'],
|
||||
);
|
||||
await Promise.all(
|
||||
[...this.completions.values()]
|
||||
.filter((entry) => ids.includes(entry.workId))
|
||||
.map((entry) => entry.done),
|
||||
);
|
||||
const rows = await this.repository.readSources([
|
||||
{ instrumentType, uid, kind: 'detail' },
|
||||
]);
|
||||
if (!rows.length) throw new Error('Instrument dependency missing');
|
||||
return (rows[0].payload as CatalogPayload).instrument ?? null;
|
||||
}
|
||||
constructor(
|
||||
private readonly repository: CatalogDiscoveryRepository,
|
||||
private readonly access: TbankAccessService,
|
||||
private readonly tbank: TbankClient,
|
||||
private readonly logger: StructuredLogger,
|
||||
private readonly moex: MoexClient,
|
||||
private readonly ratings: NcrRatingsClient,
|
||||
private readonly candles: CandleRepository,
|
||||
) {}
|
||||
|
||||
async readMany(
|
||||
userId: string,
|
||||
instrumentType: 'share' | 'bond',
|
||||
uids: string[],
|
||||
observe: boolean,
|
||||
kinds: CatalogSource['kind'][] = ['last-price', 'detail'],
|
||||
range?: {
|
||||
from: string;
|
||||
to: string;
|
||||
ticker?: string;
|
||||
tickers?: Record<string, string>;
|
||||
},
|
||||
) {
|
||||
const { revision } = await this.access.local(userId);
|
||||
const inputs = sourceInputs(instrumentType, uids, kinds, range);
|
||||
const rows = inputs.length ? await this.repository.readSources(inputs) : [];
|
||||
if (!observe)
|
||||
await this.requestMany(
|
||||
userId,
|
||||
revision,
|
||||
instrumentType,
|
||||
uids,
|
||||
false,
|
||||
kinds,
|
||||
range,
|
||||
);
|
||||
const work = inputs.length
|
||||
? await this.repository.workMany(userId, revision, inputs)
|
||||
: [];
|
||||
const byScope = new Map(work.map((row) => [row.scopeKey, row]));
|
||||
return inputs.map((input) => {
|
||||
const row = rows.find(
|
||||
(entry) =>
|
||||
entry.instrumentUid === input.uid && entry.kind === input.kind,
|
||||
);
|
||||
const payload = row?.payload as CatalogPayload | undefined;
|
||||
return {
|
||||
input,
|
||||
version: row
|
||||
? {
|
||||
instrumentUid: row.instrumentUid,
|
||||
source: row.source,
|
||||
kind: row.kind,
|
||||
rangeKey: row.rangeKey,
|
||||
dataRevision: row.dataRevision,
|
||||
fetchedAt: row.fetchedAt,
|
||||
}
|
||||
: null,
|
||||
payload,
|
||||
instrument: payload?.instrument ?? null,
|
||||
price: payload?.price ?? null,
|
||||
meta: investmentResourceMeta({
|
||||
resourceKey: catalogScope(input),
|
||||
dataRevision: String(row?.dataRevision ?? 0),
|
||||
availability: row ? 'complete' : 'missing',
|
||||
fetchedAt: row?.fetchedAt ?? null,
|
||||
freshForMs: tbankSourceTtlMs[sourcePolicy(input.kind)],
|
||||
work: byScope.get(catalogScope(input)) ?? null,
|
||||
}),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
requestMany(
|
||||
userId: string,
|
||||
revision: string,
|
||||
instrumentType: 'share' | 'bond',
|
||||
uids: string[],
|
||||
force = false,
|
||||
kinds: CatalogSource['kind'][] = ['last-price', 'detail'],
|
||||
range?: {
|
||||
from: string;
|
||||
to: string;
|
||||
ticker?: string;
|
||||
tickers?: Record<string, string>;
|
||||
},
|
||||
) {
|
||||
const inputs = sourceInputs(instrumentType, uids, kinds, range);
|
||||
const reservation = this.reservations.then(async () => {
|
||||
const rows = inputs.length
|
||||
? await this.repository.readSources(inputs)
|
||||
: [];
|
||||
const ids: string[] = [];
|
||||
for (const input of inputs) {
|
||||
const row = rows.find(
|
||||
(entry) =>
|
||||
entry.instrumentUid === input.uid && entry.kind === input.kind,
|
||||
);
|
||||
if (
|
||||
!tbankSourceNeedsRefresh(
|
||||
sourcePolicy(input.kind),
|
||||
row?.fetchedAt ?? null,
|
||||
new Date(),
|
||||
force,
|
||||
)
|
||||
)
|
||||
continue;
|
||||
const reserved = await this.repository.reserve(
|
||||
{ userId, revision },
|
||||
input,
|
||||
this.runId,
|
||||
this.active,
|
||||
);
|
||||
if (!reserved) continue;
|
||||
const { id, attemptId } = reserved.work;
|
||||
ids.push(id);
|
||||
if (!reserved.launch) continue;
|
||||
const attempt = { userId, revision, id, attemptId };
|
||||
this.active.add(attemptId);
|
||||
let resolve = () => {};
|
||||
const done = new Promise<void>((finish) => {
|
||||
resolve = finish;
|
||||
});
|
||||
this.completions.set(attemptId, { workId: id, done, resolve });
|
||||
const key = producerKey(input);
|
||||
const producer = this.producers.get(key);
|
||||
if (producer) producer.observers.push({ attempt, input });
|
||||
else {
|
||||
const queued = { input, attempt, observers: [] };
|
||||
this.producers.set(key, queued);
|
||||
this.queue.push(queued);
|
||||
}
|
||||
}
|
||||
if (this.queue.length && !this.running) {
|
||||
this.running = true;
|
||||
setImmediate(() => {
|
||||
void this.drain();
|
||||
});
|
||||
}
|
||||
return ids;
|
||||
});
|
||||
this.reservations = reservation.catch(() => undefined);
|
||||
return reservation;
|
||||
}
|
||||
|
||||
async section(
|
||||
userId: string,
|
||||
instrumentType: 'share' | 'bond',
|
||||
uid: string,
|
||||
section: 'quote' | 'company' | 'metrics',
|
||||
observe = false,
|
||||
) {
|
||||
const kinds: CatalogSource['kind'][] =
|
||||
section === 'company'
|
||||
? ['detail', 'description', 'fundamentals']
|
||||
: [
|
||||
...(instrumentType === 'bond'
|
||||
? ['bond-current' as const]
|
||||
: ['detail' as const]),
|
||||
'last-price',
|
||||
'trading-status',
|
||||
'comparison',
|
||||
...(section === 'metrics'
|
||||
? ['coupons' as const, 'yield' as const, 'rating' as const]
|
||||
: []),
|
||||
];
|
||||
const sources = await this.readMany(
|
||||
userId,
|
||||
instrumentType,
|
||||
[uid],
|
||||
observe,
|
||||
kinds,
|
||||
);
|
||||
const first = sources[0];
|
||||
if (first.meta.availability === 'complete' && !first.instrument)
|
||||
throw new NotFoundException();
|
||||
const values = sources.reduce<CatalogPayload>(
|
||||
(result, source) => ({ ...result, ...source.payload }),
|
||||
{},
|
||||
);
|
||||
const dependencies = sources.map((source) => source.meta);
|
||||
const complete = dependencies.every(
|
||||
(dependency) => dependency.availability === 'complete',
|
||||
);
|
||||
const meta = investmentResourceMeta({
|
||||
resourceKey: `${instrumentType}/${section}/${uid}`,
|
||||
dataRevision: dependencies
|
||||
.map((dependency) => dependency.dataRevision)
|
||||
.join('/'),
|
||||
availability: !first.instrument
|
||||
? 'missing'
|
||||
: complete
|
||||
? 'complete'
|
||||
: 'partial',
|
||||
fetchedAt: first.meta.lastSuccessfulAt
|
||||
? new Date(first.meta.lastSuccessfulAt)
|
||||
: null,
|
||||
freshForMs:
|
||||
section === 'company'
|
||||
? tbankSourceTtlMs.company
|
||||
: tbankSourceTtlMs.quote,
|
||||
work: null,
|
||||
dependencies,
|
||||
});
|
||||
return {
|
||||
values,
|
||||
meta,
|
||||
unavailable: sources
|
||||
.filter(
|
||||
(source) =>
|
||||
source.meta.availability !== 'complete' ||
|
||||
source.meta.sync.state === 'failed',
|
||||
)
|
||||
.map((source) => source.input.kind),
|
||||
observedAt: meta.lastSuccessfulAt ?? new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
private async load(
|
||||
input: CatalogSource,
|
||||
token: string,
|
||||
attempt: TbankSyncAttempt,
|
||||
): Promise<CatalogPayload> {
|
||||
if (input.kind === 'detail' || input.kind === 'bond-current') {
|
||||
const instrument =
|
||||
input.instrumentType === 'share'
|
||||
? await this.tbank.getShare(token, input.uid)
|
||||
: await this.tbank.getBond(token, input.uid);
|
||||
if (instrument && instrument.uid !== input.uid)
|
||||
throw new Error('Catalog UID mismatch');
|
||||
if (instrument === undefined) throw new Error('Missing catalog response');
|
||||
return { instrument: instrument ? publicInstrument(instrument) : null };
|
||||
}
|
||||
if (input.kind === 'trading-status')
|
||||
return {
|
||||
tradingStatus: await this.tbank.getTradingStatus(token, input.uid),
|
||||
};
|
||||
if (input.kind === 'comparison') {
|
||||
const prices = await this.repository.readSources([
|
||||
{ ...input, kind: 'last-price', rangeKey: undefined },
|
||||
]);
|
||||
if (!prices.length) throw new Error('Price dependency missing');
|
||||
const price = (prices[0].payload as CatalogPayload).price;
|
||||
const priceAt = timestamp(price?.time);
|
||||
if (!priceAt) return { candle: null };
|
||||
const now = new Date();
|
||||
const range = {
|
||||
from: utcDay(new Date(Date.parse(priceAt) - previousCloseLookbackMs)),
|
||||
to: new Date(priceAt),
|
||||
};
|
||||
const saved = await this.candles.read(input.uid, range);
|
||||
const gaps = candleGaps(range, saved.coverage, now);
|
||||
for (const gap of gaps) {
|
||||
const response = await this.tbank.getCandles(
|
||||
token,
|
||||
input.uid,
|
||||
gap.from,
|
||||
gap.to,
|
||||
);
|
||||
const rows = response.map((candle) => {
|
||||
const time = timestamp(candle.time);
|
||||
if (!time || !candle.close)
|
||||
throw new Error('Invalid candle response');
|
||||
return {
|
||||
time,
|
||||
close: formatDecimal(decimal(candle.close)),
|
||||
isComplete: candle.isComplete === true,
|
||||
};
|
||||
});
|
||||
if (!(await this.candles.stage(attempt, gap, rows, now)))
|
||||
throw new Error('Cancelled candle comparison');
|
||||
}
|
||||
if (
|
||||
gaps.length &&
|
||||
!(await this.candles.publish(
|
||||
attempt,
|
||||
{ uid: input.uid, ranges: gaps },
|
||||
false,
|
||||
))
|
||||
)
|
||||
throw new Error('Cancelled candle comparison');
|
||||
const values = (await this.candles.read(input.uid, range)).data.map(
|
||||
(row) => {
|
||||
const value = divide(
|
||||
decimalFromString(row.close),
|
||||
decimalFromString('1'),
|
||||
quotationScale,
|
||||
).value;
|
||||
return {
|
||||
time: {
|
||||
seconds: String(Date.parse(row.time) / MILLISECONDS_PER_SECOND),
|
||||
},
|
||||
close: {
|
||||
units: String(value / quotationUnits),
|
||||
nano: Number(value % quotationUnits),
|
||||
},
|
||||
isComplete: row.isComplete,
|
||||
};
|
||||
},
|
||||
);
|
||||
return { candle: previousClose(values, priceAt) };
|
||||
}
|
||||
const details = await this.repository.readSources([
|
||||
{ ...input, kind: 'detail', rangeKey: undefined },
|
||||
...(input.instrumentType === 'bond'
|
||||
? [{ ...input, kind: 'bond-current' as const, rangeKey: undefined }]
|
||||
: []),
|
||||
]);
|
||||
const instrument = details
|
||||
.map((row) => (row.payload as CatalogPayload).instrument)
|
||||
.find(Boolean);
|
||||
if (
|
||||
!instrument &&
|
||||
!['dividends', 'coupons', 'bond-events'].includes(input.kind)
|
||||
)
|
||||
throw new Error('Instrument dependency missing');
|
||||
if (input.kind === 'description')
|
||||
return {
|
||||
description: instrument?.assetUid
|
||||
? await this.tbank.getAssetDescription(token, instrument?.assetUid)
|
||||
: null,
|
||||
};
|
||||
if (input.kind === 'fundamentals')
|
||||
return {
|
||||
fundamentals: instrument?.assetUid
|
||||
? await this.tbank.getFundamentals(token, instrument?.assetUid)
|
||||
: null,
|
||||
};
|
||||
if (input.kind === 'coupons' || input.kind === 'dividends') {
|
||||
const year = Number(input.rangeKey);
|
||||
const [from, to] = input.rangeKey?.startsWith('[')
|
||||
? (JSON.parse(input.rangeKey) as [string, string])
|
||||
: [
|
||||
`${year - payoutHistoryYears}-01-01`,
|
||||
`${year + payoutFutureYears}-01-01`,
|
||||
];
|
||||
return input.kind === 'coupons'
|
||||
? {
|
||||
coupons: await this.tbank.getBondCoupons(
|
||||
token,
|
||||
input.uid,
|
||||
from,
|
||||
to,
|
||||
),
|
||||
}
|
||||
: {
|
||||
dividends: await this.tbank.getDividends(
|
||||
token,
|
||||
input.uid,
|
||||
from,
|
||||
to,
|
||||
),
|
||||
};
|
||||
}
|
||||
if (input.kind === 'bond-events') {
|
||||
const ticker = instrument?.ticker || input.ticker;
|
||||
if (!ticker) throw new Error('Bond ticker missing');
|
||||
return { bondEvents: await this.moex.bondEvents(ticker) };
|
||||
}
|
||||
if (input.kind === 'yield')
|
||||
return {
|
||||
effectiveYield:
|
||||
instrument?.ticker && instrument.isin && instrument.classCode
|
||||
? await this.moex.bondYield(
|
||||
instrument.ticker,
|
||||
instrument.isin,
|
||||
instrument.classCode,
|
||||
)
|
||||
: null,
|
||||
};
|
||||
if (input.kind === 'rating')
|
||||
return {
|
||||
creditRating: instrument?.isin
|
||||
? await this.ratings.rating(instrument.isin)
|
||||
: null,
|
||||
};
|
||||
throw new Error('Unsupported catalog source');
|
||||
}
|
||||
|
||||
private async drain() {
|
||||
try {
|
||||
while (this.queue.length) {
|
||||
const producer = this.queue.shift();
|
||||
if (!producer) break;
|
||||
const batch = [producer];
|
||||
// One batch RPC for queued prices sharing the same authorized producer.
|
||||
if (producer.input.kind === 'last-price') {
|
||||
for (
|
||||
let i = 0;
|
||||
i < this.queue.length && batch.length < catalogSearchLimit;
|
||||
) {
|
||||
const next = this.queue[i];
|
||||
if (
|
||||
next.input.kind === 'last-price' &&
|
||||
next.attempt.userId === producer.attempt.userId &&
|
||||
next.attempt.revision === producer.attempt.revision
|
||||
) {
|
||||
batch.push(next);
|
||||
this.queue.splice(i, 1);
|
||||
} else i++;
|
||||
}
|
||||
}
|
||||
let prices: TbankLastPrice[] | undefined;
|
||||
let payload: CatalogPayload | undefined;
|
||||
let failed = false;
|
||||
try {
|
||||
const token = await this.access.tokenForAttempt(producer.attempt);
|
||||
if (!token) failed = true;
|
||||
else if (producer.input.kind === 'last-price')
|
||||
prices = await this.tbank.getLastPrices(
|
||||
token,
|
||||
batch.map((item) => item.input.uid),
|
||||
);
|
||||
else
|
||||
payload = await this.load(producer.input, token, producer.attempt);
|
||||
} catch (error) {
|
||||
failed = true;
|
||||
if (error instanceof Error && error.message === 'INVALID_TBANK_TOKEN')
|
||||
await this.access.rejectMembershipToken(producer.attempt);
|
||||
else
|
||||
this.logger.event('warn', 'catalog-enrichment.failed', {
|
||||
code: 'TBANK_UNAVAILABLE',
|
||||
});
|
||||
}
|
||||
for (const item of batch) {
|
||||
const price =
|
||||
prices?.find((entry) => entry.instrumentUid === item.input.uid) ??
|
||||
null;
|
||||
const value =
|
||||
item.input.kind === 'last-price'
|
||||
? {
|
||||
price: price
|
||||
? {
|
||||
instrumentUid: item.input.uid,
|
||||
price: price.price,
|
||||
time: price.time,
|
||||
}
|
||||
: null,
|
||||
}
|
||||
: payload;
|
||||
// Keep accepting observers while publication awaits SQLite.
|
||||
item.observers.unshift({ attempt: item.attempt, input: item.input });
|
||||
for (const observer of item.observers) {
|
||||
const success =
|
||||
!failed &&
|
||||
value !== undefined &&
|
||||
(await this.repository.publishSource(
|
||||
observer.attempt,
|
||||
observer.input,
|
||||
value,
|
||||
new Date(),
|
||||
producer.attempt,
|
||||
));
|
||||
await this.repository.finish(observer.attempt, success);
|
||||
this.active.delete(observer.attempt.attemptId);
|
||||
this.completions.get(observer.attempt.attemptId)?.resolve();
|
||||
this.completions.delete(observer.attempt.attemptId);
|
||||
}
|
||||
this.producers.delete(producerKey(item.input));
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
for (const completion of this.completions.values()) completion.resolve();
|
||||
this.completions.clear();
|
||||
this.active.clear();
|
||||
this.producers.clear();
|
||||
this.queue.length = 0;
|
||||
this.logger.event('error', 'catalog-enrichment.persistence-failed', {
|
||||
code: 'TBANK_UNAVAILABLE',
|
||||
});
|
||||
} finally {
|
||||
this.running = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
function sourceInputs(
|
||||
instrumentType: 'share' | 'bond',
|
||||
uids: string[],
|
||||
kinds: CatalogSource['kind'][],
|
||||
range?: {
|
||||
from: string;
|
||||
to: string;
|
||||
ticker?: string;
|
||||
tickers?: Record<string, string>;
|
||||
},
|
||||
): CatalogSource[] {
|
||||
return kinds.flatMap((kind) =>
|
||||
[...new Set(uids)].map((uid) => ({
|
||||
instrumentType,
|
||||
uid,
|
||||
kind,
|
||||
...(kind === 'bond-events' && (range?.tickers?.[uid] || range?.ticker)
|
||||
? { ticker: range?.tickers?.[uid] || range?.ticker }
|
||||
: {}),
|
||||
...(['coupons', 'dividends'].includes(kind)
|
||||
? {
|
||||
rangeKey: range
|
||||
? JSON.stringify([range.from, range.to])
|
||||
: String(new Date().getUTCFullYear()),
|
||||
}
|
||||
: {}),
|
||||
})),
|
||||
);
|
||||
}
|
||||
function publicInstrument(value: TbankShare | TbankBond) {
|
||||
const bond = value as TbankBond;
|
||||
return {
|
||||
uid: value.uid,
|
||||
figi: value.figi,
|
||||
assetUid: value.assetUid,
|
||||
name: value.name,
|
||||
ticker: value.ticker,
|
||||
isin: value.isin,
|
||||
classCode: value.classCode,
|
||||
realExchange: value.realExchange,
|
||||
apiTradeAvailableFlag: value.apiTradeAvailableFlag,
|
||||
currency: value.currency,
|
||||
shareType: (value as TbankShare).shareType,
|
||||
sector: value.sector,
|
||||
lot: value.lot,
|
||||
brand: value.brand ? { logoName: value.brand.logoName } : undefined,
|
||||
first_1dayCandleDate: value.first_1dayCandleDate,
|
||||
nominal: bond.nominal,
|
||||
initialNominal: bond.initialNominal,
|
||||
aciValue: bond.aciValue,
|
||||
maturityDate: bond.maturityDate,
|
||||
callDate: bond.callDate,
|
||||
couponQuantityPerYear: bond.couponQuantityPerYear,
|
||||
floatingCouponFlag: bond.floatingCouponFlag,
|
||||
perpetualFlag: bond.perpetualFlag,
|
||||
amortizationFlag: bond.amortizationFlag,
|
||||
subordinatedFlag: bond.subordinatedFlag,
|
||||
bondType: bond.bondType,
|
||||
riskLevel: bond.riskLevel,
|
||||
};
|
||||
}
|
||||
|
||||
const previousCloseLookbackDays = 20;
|
||||
const previousCloseLookbackMs =
|
||||
previousCloseLookbackDays * MILLISECONDS_PER_DAY;
|
||||
const payoutHistoryYears = 5;
|
||||
const payoutFutureYears = 2;
|
||||
export type CatalogPayload = {
|
||||
instrument?: TbankShare | TbankBond | null;
|
||||
price?: TbankLastPrice | null;
|
||||
candle?: TbankCandle | null;
|
||||
tradingStatus?: string | null;
|
||||
description?: string | null;
|
||||
fundamentals?: TbankFundamentals | null;
|
||||
coupons?: TbankCoupon[];
|
||||
dividends?: TbankDividend[];
|
||||
bondEvents?: MoexBondEvent[];
|
||||
effectiveYield?: Awaited<ReturnType<MoexClient['bondYield']>>;
|
||||
creditRating?: Awaited<ReturnType<NcrRatingsClient['rating']>>;
|
||||
};
|
||||
function sourcePolicy(kind: CatalogSource['kind']) {
|
||||
if (
|
||||
[
|
||||
'last-price',
|
||||
'bond-current',
|
||||
'trading-status',
|
||||
'comparison',
|
||||
'yield',
|
||||
].includes(kind)
|
||||
)
|
||||
return 'quote';
|
||||
if (kind === 'description' || kind === 'fundamentals') return 'company';
|
||||
if (['coupons', 'dividends', 'bond-events'].includes(kind)) return 'calendar';
|
||||
return 'metadata';
|
||||
}
|
||||
function producerKey(input: CatalogSource) {
|
||||
return input.instrumentType === 'bond' &&
|
||||
(input.kind === 'detail' || input.kind === 'bond-current')
|
||||
? `bond-metadata:${input.uid}`
|
||||
: catalogScope(input);
|
||||
}
|
||||
|
||||
const quotationScale = 9;
|
||||
const quotationUnits = 1_000_000_000n;
|
||||
@@ -0,0 +1,39 @@
|
||||
import type { TbankShare } from '../../../../integrations/tbank/tbank.api';
|
||||
|
||||
export const catalogSearchLimit = 30;
|
||||
|
||||
export function selectCatalogInstruments<T extends TbankShare>(
|
||||
found: T[],
|
||||
query: string,
|
||||
instrumentType: 'share' | 'bond',
|
||||
): (T & { uid: string })[] {
|
||||
const seen = new Set<string>();
|
||||
const kind =
|
||||
instrumentType === 'share'
|
||||
? 'INSTRUMENT_TYPE_SHARE'
|
||||
: 'INSTRUMENT_TYPE_BOND';
|
||||
const preferredClass = instrumentType === 'share' ? 'TQBR' : 'TQOB';
|
||||
return found
|
||||
.filter(
|
||||
(instrument): instrument is T & { uid: string } =>
|
||||
Boolean(instrument.uid?.trim()) && instrument.instrumentKind === kind,
|
||||
)
|
||||
.sort(
|
||||
(a, b) =>
|
||||
Number(b.apiTradeAvailableFlag === true) -
|
||||
Number(a.apiTradeAvailableFlag === true) ||
|
||||
Number(b.ticker?.toUpperCase() === query.toUpperCase()) -
|
||||
Number(a.ticker?.toUpperCase() === query.toUpperCase()) ||
|
||||
Number(b.classCode === preferredClass) -
|
||||
Number(a.classCode === preferredClass) ||
|
||||
a.uid.localeCompare(b.uid),
|
||||
)
|
||||
.filter((instrument) => {
|
||||
const key = instrument.isin?.trim()
|
||||
? `isin:${instrument.isin.trim().toUpperCase()}`
|
||||
: `uid:${instrument.uid}`;
|
||||
if (seen.has(key)) return false;
|
||||
seen.add(key);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user