feat: add hourly portfolio sync history
Backend verify / commits (pull_request) Successful in 33s
Backend verify / security (pull_request) Failing after 12s
Backend verify / verify (pull_request) Skipped

This commit is contained in:
2026-10-09 09:04:41 +03:00
parent 03e75f8d2b
commit 0057925888
49 changed files with 1674 additions and 1870 deletions
@@ -2,6 +2,7 @@ import { ApiProperty } from '@nestjs/swagger';
import { MoneyDto } from '../../../common/money.dto';
import { InvestmentResourceMetaDto } from '../investment-resource.dto';
import { HourlySyncStatusDto } from '../tbank-sync/tbank-sync.dto';
export class PortfolioSummaryDto {
@ApiProperty({ type: MoneyDto, isArray: true })
@@ -55,4 +56,12 @@ export class BrokerageAccountsResourceDto {
@ApiProperty({ type: InvestmentResourceMetaDto })
declare meta: InvestmentResourceMetaDto;
@ApiProperty({
type: HourlySyncStatusDto,
nullable: true,
description:
'Read-only status of the last hourly walk for the current connection. Null when no walk attempt exists yet.',
})
declare sync: HourlySyncStatusDto | null;
}
@@ -1,11 +1,10 @@
import type { Request } from 'express';
import { Controller, Get, Query, Req, UseGuards } from '@nestjs/common';
import { Controller, Get, Req, UseGuards } from '@nestjs/common';
import { ApiBearerAuth, ApiOkResponse, ApiTags } from '@nestjs/swagger';
import { ApiProblemResponses } from '../../../common/decorators/api-safe-error-response.decorator';
import { AccessPayload, JwtAuthGuard } from '../../auth/auth.api';
import { InvestmentReadQueryDto } from '../investment-resource.dto';
import { BrokerageAccountsResourceDto } from './brokerage-account.dto';
import { BrokerageAccountsService } from './brokerage-accounts.service';
@@ -19,10 +18,7 @@ export class BrokerageAccountsController {
@Get()
@ApiOkResponse({ type: BrokerageAccountsResourceDto })
list(
@Req() request: Request & { user: AccessPayload },
@Query() query: InvestmentReadQueryDto,
) {
return this.accounts.list(request.user.sub, query.sync === 'observe');
list(@Req() request: Request & { user: AccessPayload }) {
return this.accounts.list(request.user.sub);
}
}
@@ -3,11 +3,12 @@ import { Module } from '@nestjs/common';
import { TbankModule } from '../../../integrations/tbank/tbank.module';
import { AuthModule } from '../../auth/auth.module';
import { PortfolioModule } from '../portfolio/portfolio.module';
import { TbankSyncModule } from '../tbank-sync/tbank-sync.module';
import { BrokerageAccountsController } from './brokerage-accounts.controller';
import { BrokerageAccountsService } from './brokerage-accounts.service';
@Module({
imports: [AuthModule, TbankModule, PortfolioModule],
imports: [AuthModule, TbankModule, PortfolioModule, TbankSyncModule],
controllers: [BrokerageAccountsController],
providers: [BrokerageAccountsService],
exports: [BrokerageAccountsService],
@@ -19,16 +19,17 @@ const meta = investmentResourceMeta({
describe('saved brokerage account projection', () => {
const portfolio = { saved: jest.fn() };
const service = new BrokerageAccountsService(portfolio as never);
const sync = { hourly: jest.fn().mockResolvedValue(null) };
const service = new BrokerageAccountsService(portfolio as never, sync as never);
it('distinguishes unknown membership from a confirmed empty list', async () => {
portfolio.saved.mockResolvedValueOnce({
membership: { ...meta, availability: 'missing' },
accounts: [],
});
expect((await service.list('owner', true)).data).toBeNull();
expect((await service.list('owner')).data).toBeNull();
portfolio.saved.mockResolvedValueOnce({ membership: meta, accounts: [] });
expect((await service.list('owner', true)).data).toEqual([]);
expect((await service.list('owner')).data).toEqual([]);
});
it('keeps exact saved totals and independent missing summaries', async () => {
@@ -58,7 +59,7 @@ describe('saved brokerage account projection', () => {
},
],
});
const response = await service.list('owner', true);
const response = await service.list('owner');
expect(response.data?.[0].portfolioSummary).toEqual({
totalsByCurrency: [{ currency: 'RUB', value: '100000000.000000001' }],
stockPositionCount: 1,
@@ -3,14 +3,18 @@ import { createHash } from 'node:crypto';
import { investmentResourceMeta } from '../investment-resource.mapper';
import { PortfolioService } from '../portfolio/portfolio.api';
import { BrokerageAccountDto } from './brokerage-account.dto';
import { TbankSyncRepository } from '../tbank-sync/tbank-sync.repository';
import { BrokerageAccountDto, BrokerageAccountsResourceDto } from './brokerage-account.dto';
@Injectable()
export class BrokerageAccountsService {
constructor(private readonly portfolio: PortfolioService) {}
constructor(
private readonly portfolio: PortfolioService,
private readonly sync: TbankSyncRepository,
) {}
async list(userId: string, observe = false) {
const saved = await this.portfolio.saved(userId, 'all', observe);
async list(userId: string): Promise<BrokerageAccountsResourceDto> {
const saved = await this.portfolio.saved(userId, 'all');
const data: BrokerageAccountDto[] | null =
saved.membership.availability === 'missing'
? null
@@ -31,6 +35,8 @@ export class BrokerageAccountsService {
}
: null,
}));
// The read-only result of the last hourly walk; never starts one.
const hourly = await this.sync.hourly(userId);
return {
data,
meta: investmentResourceMeta({
@@ -55,12 +61,20 @@ export class BrokerageAccountsService {
fetchedAt: saved.membership.lastSuccessfulAt
? new Date(saved.membership.lastSuccessfulAt)
: null,
freshForMs: 300_000,
freshForMs: 3_600_000,
work: saved.membership.sync.id
? { ...saved.membership.sync, id: saved.membership.sync.id }
: null,
dependencies: saved.accounts.map((account) => account.meta),
}),
sync: hourly
? {
state: hourly.state,
startedAt: hourly.startedAt.toISOString(),
finishedAt: hourly.finishedAt?.toISOString() ?? null,
sections: hourly.sections,
}
: null,
};
}
}
@@ -9,8 +9,9 @@ import { StructuredLogger } from '../../../../common/logging/structured-logger.s
import { isTbankErrorCode, TbankErrorCode } from '../../../../integrations/tbank/tbank-error-code';
import {
TbankAccessService,
type TbankBond,
TbankClient,
tbankSourceNeedsRefresh,
type TbankShare,
tbankSourceTtlMs,
type TbankSyncAttempt,
} from '../../../../integrations/tbank/tbank.api';
@@ -31,6 +32,7 @@ import {
const UNAMBIGUOUS_MATCH_COUNT = 1;
const NO_MATCHES = 0;
const FIRST_RESULT_OFFSET = 0;
const MISSING_DATA_REVISION = '0';
const tickerAmbiguityLimit = 2;
@@ -73,13 +75,14 @@ export class CatalogDiscoveryService {
dataRevision: saved.dataRevision,
availability: saved.data === null ? 'missing' : 'complete',
fetchedAt: saved.fetchedAt,
freshForMs: tbankSourceTtlMs.catalog,
freshForMs: Number.MAX_SAFE_INTEGER,
work: await this.repository.work(userId, connection.revision, input),
}),
};
}
async search(userId: string, input: CatalogQuery, observe = false) {
if (!observe) return this.onlineSearch(userId, catalogQuery(input));
const discovery = await this.read(userId, input, observe);
const sources = discovery.data
? await this.enrichment.readMany(
@@ -124,7 +127,7 @@ export class CatalogDiscoveryService {
fetchedAt: discovery.meta.lastSuccessfulAt
? new Date(discovery.meta.lastSuccessfulAt)
: null,
freshForMs: tbankSourceTtlMs.catalog,
freshForMs: Number.MAX_SAFE_INTEGER,
work: discovery.meta.sync.id
? {
id: discovery.meta.sync.id,
@@ -143,6 +146,25 @@ export class CatalogDiscoveryService {
uid: string,
observe = false,
) {
if (!observe) {
const token = await this.access.onlineToken(userId);
const instrument =
instrumentType === 'share'
? await this.tbank.getShare(token, uid)
: await this.tbank.getBond(token, uid);
if (!instrument) throw new NotFoundException();
return {
data: { ...publicInstrument({ ...instrument, uid }), uid, slug: instrument.ticker?.toUpperCase() ?? uid },
meta: investmentResourceMeta({
resourceKey: `${instrumentType}/detail/${uid}`,
dataRevision: `online:${Date.now()}`,
availability: 'complete',
fetchedAt: new Date(),
freshForMs: tbankSourceTtlMs.portfolio,
work: null,
}),
};
}
const sources = await this.enrichment.readMany(
userId,
instrumentType,
@@ -182,7 +204,7 @@ export class CatalogDiscoveryService {
fetchedAt: source.meta.lastSuccessfulAt
? new Date(source.meta.lastSuccessfulAt)
: null,
freshForMs: tbankSourceTtlMs.metadata,
freshForMs: tbankSourceTtlMs.portfolio,
work: source.meta.sync.id
? {
id: source.meta.sync.id,
@@ -195,12 +217,37 @@ export class CatalogDiscoveryService {
};
}
// eslint-disable-next-line sonarjs/cognitive-complexity -- Online and observe paths intentionally preserve distinct error semantics.
async byTicker(
userId: string,
instrumentType: 'bond' | 'share',
ticker: string,
observe = false,
) {
if (!observe) {
const query = ticker.trim().toUpperCase();
const token = await this.access.onlineToken(userId);
const found =
instrumentType === 'share'
? await this.tbank.findShares(token, query)
: await this.tbank.findBonds(token, query);
const matches = selectCatalogInstruments(
found.filter((item) => item.ticker?.trim().toUpperCase() === query),
query,
instrumentType,
).slice(FIRST_RESULT_OFFSET, tickerAmbiguityLimit);
if (!matches.length) throw new NotFoundException();
if (matches.length > UNAMBIGUOUS_MATCH_COUNT) {
throw new ConflictException({
code:
instrumentType === 'share'
? InstrumentErrorCode.StockTickerAmbiguous
: InstrumentErrorCode.BondTickerAmbiguous,
});
}
const result = await this.detail(userId, instrumentType, matches[0].uid);
return { ...result, data: result.data ? { ...result.data, slug: query } : null };
}
const lookup = await this.read(
userId,
{ instrumentType, query: ticker, exactTicker: true },
@@ -232,7 +279,7 @@ export class CatalogDiscoveryService {
fetchedAt: lookup.meta.lastSuccessfulAt
? new Date(lookup.meta.lastSuccessfulAt)
: null,
freshForMs: tbankSourceTtlMs.metadata,
freshForMs: tbankSourceTtlMs.portfolio,
work: lookup.meta.sync.id
? {
id: lookup.meta.sync.id,
@@ -245,6 +292,51 @@ export class CatalogDiscoveryService {
};
}
private async onlineSearch(userId: string, input: CatalogQuery) {
const token = await this.access.onlineToken(userId);
const found =
input.instrumentType === 'share'
? await this.tbank.findShares(token, input.query)
: await this.tbank.findBonds(token, input.query);
const instruments = selectCatalogInstruments(
input.exactTicker
? found.filter((item) => item.ticker?.trim().toUpperCase() === input.query)
: found,
input.query,
input.instrumentType,
).slice(FIRST_RESULT_OFFSET, input.exactTicker ? tickerAmbiguityLimit : catalogSearchLimit);
const details = await Promise.all(
instruments.map((item) =>
input.instrumentType === 'share'
? this.tbank.getShare(token, item.uid)
: this.tbank.getBond(token, item.uid),
),
);
const prices = instruments.length
? await this.tbank.getLastPrices(token, instruments.map((item) => item.uid))
: [];
const fetchedAt = new Date();
const data = instruments.flatMap((item, index) => {
const detail = details.at(index);
return [{
...item,
detail: detail ? publicInstrument({ ...detail, uid: item.uid }) : null,
price: prices.find((price) => price.instrumentUid === item.uid) ?? null,
}];
});
return {
data,
meta: investmentResourceMeta({
resourceKey: `search:${catalogScope(input)}`,
dataRevision: `online:${fetchedAt.toISOString()}`,
availability: 'complete',
fetchedAt,
freshForMs: tbankSourceTtlMs.portfolio,
work: null,
}),
};
}
request(
userId: string,
revision: string,
@@ -255,10 +347,9 @@ export class CatalogDiscoveryService {
const reserved = this.reservations.then(async () => {
const saved = await this.repository.read(input);
if (
!tbankSourceNeedsRefresh('catalog', saved.fetchedAt, new Date(), force)
) {
return null;
}
!force &&
saved.dataRevision !== MISSING_DATA_REVISION
) return null;
const reservation = await this.repository.reserve(
{ userId, revision },
input,
@@ -404,3 +495,37 @@ export class CatalogDiscoveryService {
}
}
}
function publicInstrument(value: TbankBond | TbankShare) {
const bond = value as TbankBond;
const share = value as TbankShare;
return {
uid: value.uid,
figi: value.figi,
assetUid: value.assetUid,
name: value.name,
ticker: value.ticker,
isin: value.isin,
classCode: value.classCode,
realExchange: value.realExchange,
apiTradeAvailableFlag: value.apiTradeAvailableFlag,
currency: value.currency,
shareType: share.shareType,
sector: value.sector,
lot: value.lot,
brand: value.brand ? { logoName: value.brand.logoName } : undefined,
first_1dayCandleDate: value.first_1dayCandleDate,
nominal: bond.nominal,
initialNominal: bond.initialNominal,
aciValue: bond.aciValue,
maturityDate: bond.maturityDate,
callDate: bond.callDate,
couponQuantityPerYear: bond.couponQuantityPerYear,
floatingCouponFlag: bond.floatingCouponFlag,
perpetualFlag: bond.perpetualFlag,
amortizationFlag: bond.amortizationFlag,
subordinatedFlag: bond.subordinatedFlag,
bondType: bond.bondType,
riskLevel: bond.riskLevel,
};
}
@@ -154,7 +154,7 @@ export class CatalogEnrichmentService {
dataRevision: String(row?.dataRevision ?? MISSING_DATA_REVISION),
availability: row ? 'complete' : 'missing',
fetchedAt: row?.fetchedAt ?? null,
freshForMs: tbankSourceTtlMs[sourcePolicy(input.kind)],
freshForMs: sourcePolicy(input.kind) === 'quote' ? tbankSourceTtlMs.quote : Number.MAX_SAFE_INTEGER,
work: byScope.get(catalogScope(input)) ?? null,
}),
};
@@ -186,14 +186,10 @@ export class CatalogEnrichmentService {
(entry) =>
entry.instrumentUid === input.uid && entry.kind === input.kind,
);
if (
!tbankSourceNeedsRefresh(
sourcePolicy(input.kind),
row?.fetchedAt ?? null,
new Date(),
force,
)
) {
const policy = sourcePolicy(input.kind);
const needsRefresh = force || !row?.fetchedAt ||
(policy === 'quote' && tbankSourceNeedsRefresh('quote', row.fetchedAt, new Date()));
if (!needsRefresh) {
continue;
}
const reserved = await this.repository.reserve(
@@ -234,6 +230,7 @@ export class CatalogEnrichmentService {
return reservation;
}
// eslint-disable-next-line sonarjs/cognitive-complexity -- Section composition owns the source dependency policy.
async section(
userId: string,
instrumentType: 'bond' | 'share',
@@ -241,6 +238,34 @@ export class CatalogEnrichmentService {
section: 'company' | 'metrics' | 'quote',
observe = false,
) {
if (section === 'company' && !observe) {
const token = await this.access.onlineToken(userId);
const instrument =
instrumentType === 'share'
? await this.tbank.getShare(token, uid)
: await this.tbank.getBond(token, uid);
if (!instrument) throw new NotFoundException();
const [description, fundamentals] = instrument.assetUid
? await Promise.all([
this.tbank.getAssetDescription(token, instrument.assetUid),
this.tbank.getFundamentals(token, instrument.assetUid),
])
: [null, null];
const meta = investmentResourceMeta({
resourceKey: `${instrumentType}/company/${uid}`,
dataRevision: `online:${Date.now()}`,
availability: 'complete',
fetchedAt: new Date(),
freshForMs: Number.MAX_SAFE_INTEGER,
work: null,
});
return {
values: { instrument: publicInstrument({ ...instrument, uid }), description, fundamentals },
meta,
unavailable: [],
observedAt: meta.lastSuccessfulAt ?? new Date().toISOString(),
};
}
const kinds: CatalogSource['kind'][] =
section === 'company'
? ['detail', 'description', 'fundamentals']
@@ -287,10 +312,7 @@ export class CatalogEnrichmentService {
fetchedAt: first.meta.lastSuccessfulAt
? new Date(first.meta.lastSuccessfulAt)
: null,
freshForMs:
section === 'company'
? tbankSourceTtlMs.company
: tbankSourceTtlMs.quote,
freshForMs: section === 'company' ? Number.MAX_SAFE_INTEGER : tbankSourceTtlMs.quote,
work: null,
dependencies,
});
@@ -308,6 +330,7 @@ export class CatalogEnrichmentService {
};
}
// eslint-disable-next-line sonarjs/cognitive-complexity -- Source loading intentionally dispatches by persisted resource kind.
private async load(
input: CatalogSource,
token: string,
@@ -323,7 +346,7 @@ export class CatalogEnrichmentService {
}
// eslint-disable-next-line sonarjs/different-types-comparison -- Defensive validation rejects missing upstream payloads instead of caching them as absence.
if (instrument === undefined) throw new Error('Missing catalog response');
return { instrument: instrument ? publicInstrument(instrument) : null };
return { instrument: instrument ? publicInstrument({ ...instrument, uid: input.uid }) : null };
}
if (input.kind === 'trading-status') {
return {
@@ -470,6 +493,7 @@ export class CatalogEnrichmentService {
throw new Error('Unsupported catalog source');
}
// eslint-disable-next-line sonarjs/cognitive-complexity -- Queue draining coordinates publication fences and observer completion.
private async drain() {
try {
while (this.queue.length) {
@@ -600,39 +624,6 @@ function sourceInputs(
})),
);
}
function publicInstrument(value: TbankBond | TbankShare) {
const bond = value as TbankBond;
return {
uid: value.uid,
figi: value.figi,
assetUid: value.assetUid,
name: value.name,
ticker: value.ticker,
isin: value.isin,
classCode: value.classCode,
realExchange: value.realExchange,
apiTradeAvailableFlag: value.apiTradeAvailableFlag,
currency: value.currency,
shareType: (value as TbankShare).shareType,
sector: value.sector,
lot: value.lot,
brand: value.brand ? { logoName: value.brand.logoName } : undefined,
first_1dayCandleDate: value.first_1dayCandleDate,
nominal: bond.nominal,
initialNominal: bond.initialNominal,
aciValue: bond.aciValue,
maturityDate: bond.maturityDate,
callDate: bond.callDate,
couponQuantityPerYear: bond.couponQuantityPerYear,
floatingCouponFlag: bond.floatingCouponFlag,
perpetualFlag: bond.perpetualFlag,
amortizationFlag: bond.amortizationFlag,
subordinatedFlag: bond.subordinatedFlag,
bondType: bond.bondType,
riskLevel: bond.riskLevel,
};
}
const previousCloseLookbackDays = 20;
const previousCloseLookbackMs =
previousCloseLookbackDays * MILLISECONDS_PER_DAY;
@@ -651,6 +642,40 @@ export type CatalogPayload = {
price?: null | TbankLastPrice;
tradingStatus?: null | string;
};
function publicInstrument(value: TbankBond | TbankShare) {
const bond = value as TbankBond;
const share = value as TbankShare;
return {
uid: value.uid,
figi: value.figi,
assetUid: value.assetUid,
name: value.name,
ticker: value.ticker,
isin: value.isin,
classCode: value.classCode,
realExchange: value.realExchange,
apiTradeAvailableFlag: value.apiTradeAvailableFlag,
currency: value.currency,
shareType: share.shareType,
sector: value.sector,
lot: value.lot,
brand: value.brand ? { logoName: value.brand.logoName } : undefined,
first_1dayCandleDate: value.first_1dayCandleDate,
nominal: bond.nominal,
initialNominal: bond.initialNominal,
aciValue: bond.aciValue,
maturityDate: bond.maturityDate,
callDate: bond.callDate,
couponQuantityPerYear: bond.couponQuantityPerYear,
floatingCouponFlag: bond.floatingCouponFlag,
perpetualFlag: bond.perpetualFlag,
amortizationFlag: bond.amortizationFlag,
subordinatedFlag: bond.subordinatedFlag,
bondType: bond.bondType,
riskLevel: bond.riskLevel,
};
}
function sourcePolicy(kind: CatalogSource['kind']) {
if (
[
@@ -663,9 +688,7 @@ function sourcePolicy(kind: CatalogSource['kind']) {
) {
return 'quote';
}
if (kind === 'description' || kind === 'fundamentals') return 'company';
if (['bond-events', 'coupons', 'dividends'].includes(kind)) return 'calendar';
return 'metadata';
return 'stable';
}
function producerKey(input: CatalogSource) {
return input.instrumentType === 'bond' &&
@@ -71,8 +71,8 @@ export class BondIncomeService {
};
}
async readPosition(userId: string, uid: string, observe = false) {
const saved = await this.portfolios.read(userId, 'all', observe);
async readPosition(userId: string, uid: string) {
const saved = await this.portfolios.read(userId, 'all');
return {
data: saved.data ? instrumentPosition(saved.data, uid, 'BOND') : null,
meta: { ...saved.meta, resourceKey: `bond/position/${uid}` },
@@ -88,7 +88,7 @@ export class BondIncomeService {
const today = moscowDay(new Date().toISOString());
const from = `${Number(today.slice(ISO_DATE_START, ISO_YEAR_LENGTH)) - INCOME_HISTORY_YEARS}${today.slice(ISO_YEAR_LENGTH)}`;
const to = `${Number(today.slice(ISO_DATE_START, ISO_YEAR_LENGTH)) + INCOME_FORECAST_YEARS}${today.slice(ISO_YEAR_LENGTH)}`;
const portfolio = await this.portfolios.read(userId, 'all', observe);
const portfolio = await this.portfolios.read(userId, 'all');
const sources = await this.sources.readMany(
userId,
'bond',
@@ -119,13 +119,8 @@ export class BondController {
position(
@Req() request: Request & { user: AccessPayload },
@Param('uid', new ParseUUIDPipe()) uid: string,
@Query() query: InvestmentReadQueryDto,
) {
return this.income.readPosition(
request.user.sub,
uid,
query.sync === 'observe',
);
return this.income.readPosition(request.user.sub, uid);
}
@Get(':uid/coupons')
@@ -40,8 +40,8 @@ export class StockIncomeService {
private readonly forecasts: ForecastService,
) {}
async readPosition(userId: string, uid: string, observe = false) {
const saved = await this.portfolios.read(userId, 'all', observe);
async readPosition(userId: string, uid: string) {
const saved = await this.portfolios.read(userId, 'all');
return {
data: saved.data ? instrumentPosition(saved.data, uid, 'STOCK') : null,
meta: { ...saved.meta, resourceKey: `stock/position/${uid}` },
@@ -57,7 +57,7 @@ export class StockIncomeService {
const today = moscowDay(new Date().toISOString());
const from = `${Number(today.slice(ISO_DATE_START, ISO_YEAR_LENGTH)) - INCOME_HISTORY_YEARS}${today.slice(ISO_YEAR_LENGTH)}`;
const to = `${Number(today.slice(ISO_DATE_START, ISO_YEAR_LENGTH)) + INCOME_FORECAST_YEARS}${today.slice(ISO_YEAR_LENGTH)}`;
const portfolio = await this.portfolios.read(userId, 'all', observe);
const portfolio = await this.portfolios.read(userId, 'all');
const sources = await this.sources.readMany(
userId,
'share',
@@ -111,13 +111,8 @@ export class StockController {
position(
@Req() request: Request & { user: AccessPayload },
@Param('uid', new ParseUUIDPipe()) uid: string,
@Query() query: InvestmentReadQueryDto,
) {
return this.income.readPosition(
request.user.sub,
uid,
query.sync === 'observe',
);
return this.income.readPosition(request.user.sub, uid);
}
@Get(':uid/company')
@@ -135,50 +135,14 @@ export class BrokerageOperationsService {
? [{ accountId: source.accountId, from: source.from, to: source.to }]
: [],
);
if (query.sync !== 'observe') {
const discovery = await this.membership.request(
userId,
connection.revision,
);
if (discovery) {
void discovery.done
.then(async () => {
const current = await this.access.local(userId, accountId);
if (current.revision !== connection.revision) return;
const currentRange = ledgerRange(
query.from,
query.to,
query.instrumentUid,
current.accounts.filter(
(account) =>
accountId === 'all' || account.accountId === accountId,
),
now,
);
const targets = current.accounts
.filter(
(account) =>
accountId === 'all' || account.accountId === accountId,
)
.flatMap((account) =>
currentRange.from < currentRange.to
? [{ accountId: account.accountId, ...currentRange }]
: [],
);
await this.sync.readPlan(userId, targets, {
movingEnd,
history: true,
revision: connection.revision,
});
})
.catch(() => undefined);
} else if (intervals.length) {
await this.sync.readPlan(userId, intervals, {
movingEnd,
history: true,
revision: connection.revision,
});
}
// The ledger only starts the first-time load of missing intervals; the
// mutable tail belongs to the hourly walk and membership is not rechecked.
if (intervals.length) {
await this.sync.readPlan(userId, intervals, {
movingEnd,
history: true,
revision: connection.revision,
});
}
const saved = await this.repository.readPage(readInput);
const accounts = saved?.connection.accounts ?? [];
@@ -252,26 +216,10 @@ export class BrokerageOperationsService {
dataRevision: value.dataRevision,
availability: 'complete',
fetchedAt: value.fetchedAt,
freshForMs: tbankSourceTtlMs.metadata,
freshForMs: tbankSourceTtlMs.portfolio,
work: null,
}),
);
if (query.sync !== 'observe') {
const identities = new Map(
rows.map((row) => {
const identity = row.payload as TbankOperationItem;
return [
metadataKey(identity),
{ instrumentUid: identity.instrumentUid, figi: identity.figi },
] as const;
}),
);
for (const identity of identities.values()) {
if (!instruments.get(metadataKey(identity))?.fresh) {
await this.metadata.request(userId, connection.revision, identity);
}
}
}
const membershipMeta = investmentResourceMeta({
resourceKey: 'accounts/membership',
dataRevision: String(saved?.connection.membershipRevision ?? MISSING_MEMBERSHIP_REVISION),
@@ -71,7 +71,7 @@ describe('operation completeness and freshness', () => {
const asOf = new Date('2026-09-25T12:00:00Z');
const covered = { accountId: 'a', from, to: asOf, asOf, fetchedAt: asOf };
it.each([1, 299, 300])(
it.each([1, 3599, 3600])(
'preserves the observed complete end after %s seconds, separately from TTL',
(seconds) => {
const now = new Date(asOf.valueOf() + seconds * 1000);
@@ -85,10 +85,10 @@ describe('operation completeness and freshness', () => {
availability: 'complete',
asOf,
to: asOf,
stale: seconds >= 300,
stale: seconds >= 3600,
});
expect(plan.load).toEqual(
seconds >= 300 ? [{ accountId: 'a', from, to: now }] : [],
[],
);
},
);
@@ -128,7 +128,7 @@ export type OperationCoverage = OperationInterval & {
asOf: Date;
fetchedAt: Date;
};
const operationsFreshForMs = 300_000;
const operationsFreshForMs = 3_600_000;
export function planOperationRead(
requested: OperationInterval[],
@@ -188,8 +188,9 @@ export function planOperationRead(
.filter((row) => row.from < row.to),
fresh.map((row) => ({ ...row, instrumentUid: target.instrumentUid })),
).length > EMPTY_INTERVAL_COUNT;
// The mutable tail is owned by the hourly walk; reads only load what is
// missing, never re-verify complete or fresh intervals.
if (options.force) load.push(target);
else if (stale) load.push({ ...target, from: monthStart, to: now });
else load.push(...missing);
// Completeness is bounded by the observed end, never by the new wall clock.
return {
@@ -15,10 +15,7 @@ import {
import { MoneyDto } from '../../../../common/money.dto';
import { AccountReferenceDto } from '../../account-reference.dto';
import {
InvestmentReadQueryDto,
InvestmentResourceMetaDto,
} from '../../investment-resource.dto';
import { InvestmentResourceMetaDto } from '../../investment-resource.dto';
const MIN_OPERATIONS_PAGE_SIZE = 1;
@@ -36,7 +33,7 @@ export const operationCategories = [
] as const;
export type OperationCategory = (typeof operationCategories)[number];
export class BrokerageOperationsQueryDto extends InvestmentReadQueryDto {
export class BrokerageOperationsQueryDto {
@ApiPropertyOptional({
format: 'uuid',
description:
@@ -40,6 +40,7 @@ describe('persistent portfolio projection', () => {
accounts: [],
work: [],
});
repository.publish.mockResolvedValue(true);
metadata.savedMany.mockResolvedValue(new Map());
metadata.request.mockResolvedValue(null);
});
@@ -95,7 +96,7 @@ describe('persistent portfolio projection', () => {
],
work: [],
});
await service.read('user', 'a');
await service.request('user', 'private', 'a');
for (
let index = 0;
index < 10 && !repository.publish.mock.calls.length;
@@ -189,10 +190,10 @@ describe('persistent portfolio projection', () => {
accounts: [account, { ...account, accountId: 'b', portfolio: null }],
work: [],
});
const partial = await service.read('user', 'all', true);
const partial = await service.read('user', 'all');
expect(partial.data).toBeNull();
expect(partial.meta.availability).toBe('partial');
const saved = await service.saved('user', 'all', true);
const saved = await service.saved('user', 'all');
expect(saved.accounts[0].snapshot?.totalsByCurrency).toEqual(
snapshot.totalsByCurrency,
);
@@ -210,7 +211,7 @@ describe('persistent portfolio projection', () => {
],
work: [],
});
const complete = await service.read('user', 'all', true);
const complete = await service.read('user', 'all');
expect(complete.data?.totalsByCurrency).toEqual([
{ currency: 'RUB', value: '2.000000002' },
]);
@@ -8,7 +8,6 @@ import { StructuredLogger } from '../../../../common/logging/structured-logger.s
import { isTbankErrorCode, TbankErrorCode } from '../../../../integrations/tbank/tbank-error-code';
import {
InstrumentMetadataService,
type MetadataIdentity,
metadataKey,
TbankAccessService,
TbankClient,
@@ -47,47 +46,9 @@ export class PortfolioService {
private readonly logger: StructuredLogger,
) {}
async saved(userId: string, accountId: string, observe = false) {
// Local reads never start upstream work; the hourly walk owns refreshes.
async saved(userId: string, accountId: string) {
const connection = await this.access.local(userId, accountId);
if (!observe) {
const discovery = await this.membership.request(
userId,
connection.revision,
);
if (discovery) {
void discovery.done
.then(async () => {
const current = await this.access.local(userId, accountId);
if (current.revision !== connection.revision) return;
for (const account of current.accounts.filter(
(account) =>
accountId === 'all' || account.accountId === accountId,
)) {
await this.queueSnapshot(
userId,
current.revision,
account.accountId,
);
}
})
.catch(() =>
this.logger.event(
'warn',
'tbank.portfolio.discovery_not_available',
{ userId },
),
);
}
for (const account of (discovery ? [] : connection.accounts).filter(
(account) => accountId === 'all' || account.accountId === accountId,
)) {
await this.queueSnapshot(
userId,
connection.revision,
account.accountId,
);
}
}
const saved = await this.repository.read(userId, connection.revision);
if (
saved?.membershipAt &&
@@ -96,26 +57,6 @@ export class PortfolioService {
) {
throw new NotFoundException();
}
if (!observe) {
const identities = new Map<string, MetadataIdentity>();
for (const account of (saved?.accounts ?? []).filter(
(account) => accountId === 'all' || account.accountId === accountId,
)) {
const snapshot = account.portfolio?.payload as unknown as
PortfolioDto | undefined;
for (const position of snapshot?.positions ?? []) {
identities.set(metadataKey(position), position);
}
}
const savedMetadata = await this.metadata.savedMany([
...identities.values(),
]);
for (const identity of identities.values()) {
if (!savedMetadata.get(metadataKey(identity))?.fresh) {
await this.metadata.request(userId, connection.revision, identity);
}
}
}
const membershipMeta = investmentResourceMeta({
resourceKey: 'accounts/membership',
dataRevision: String(saved?.membershipRevision ?? MISSING_DATA_REVISION),
@@ -159,8 +100,8 @@ export class PortfolioService {
};
}
async read(userId: string, accountId: string, observe = false) {
const saved = await this.saved(userId, accountId, observe);
async read(userId: string, accountId: string) {
const saved = await this.saved(userId, accountId);
const missing =
saved.membership.availability === 'missing' ||
saved.accounts.some((account) => !account.snapshot);
@@ -176,7 +117,7 @@ export class PortfolioService {
dataRevision: source.dataRevision,
availability: 'complete',
fetchedAt: source.fetchedAt,
freshForMs: tbankSourceTtlMs.metadata,
freshForMs: portfolioFreshForMs,
work: null,
}),
]
@@ -69,17 +69,23 @@ export class PortfolioRepository {
const snapshot = await tx.tbankPortfolio.findUnique({
where: { userId_revision_accountId: { userId, revision, accountId } },
});
const portfolioWorkScope = {
userId,
revision,
scopeKey: portfolioWorkKey(accountId),
};
const previous = await tx.tbankSyncWork.findUnique({
where: { userId_revision_scopeKey: portfolioWorkScope },
});
if (
!force &&
snapshot &&
!tbankSourceNeedsRefresh('portfolio', snapshot.fetchedAt, new Date())
!tbankSourceNeedsRefresh('portfolio', snapshot.fetchedAt, new Date()) &&
previous?.state !== 'failed'
) {
return null;
}
const scope = { userId, revision, scopeKey: portfolioWorkKey(accountId) };
const previous = await tx.tbankSyncWork.findUnique({
where: { userId_revision_scopeKey: scope },
});
const scope = portfolioWorkScope;
if (
previous?.state === 'running' &&
previous.processRunId === processRunId &&
@@ -139,6 +145,41 @@ export class PortfolioRepository {
create: { ...scope, ...data },
update: { ...data, dataRevision: { increment: 1 } },
});
// Value history points require a fully valued snapshot; the attempt
// identity deduplicates repeated or late publications of one attempt.
if (
snapshot.totalsByCurrency.length > Number.MIN_VALUE &&
snapshot.positions.every((position) => position.currentValue !== null)
) {
await tx.tbankPortfolioValueHistory.upsert({
where: {
userId_accountId: { userId: attempt.userId, accountId },
},
create: { userId: attempt.userId, accountId },
update: { deletedAt: null },
});
for (const { currency, value } of snapshot.totalsByCurrency) {
await tx.tbankPortfolioValuePoint.upsert({
where: {
userId_accountId_attemptId_currency: {
userId: attempt.userId,
accountId,
attemptId: attempt.attemptId,
currency,
},
},
create: {
userId: attempt.userId,
accountId,
attemptId: attempt.attemptId,
currency,
value,
capturedAt: data.fetchedAt,
},
update: {},
});
}
}
return true;
});
}
@@ -1,11 +1,10 @@
import type { Request } from 'express';
import { Controller, Get, Param, Query, Req, UseGuards } from '@nestjs/common';
import { Controller, Get, Param, Req, UseGuards } from '@nestjs/common';
import { ApiBearerAuth, ApiOkResponse, ApiTags } from '@nestjs/swagger';
import { ApiProblemResponses } from '../../../../common/decorators/api-safe-error-response.decorator';
import { AccessPayload, JwtAuthGuard } from '../../../auth/auth.api';
import { InvestmentReadQueryDto } from '../../investment-resource.dto';
import { PortfolioService } from '../application/portfolio.service';
import { PortfolioResourceDto } from './portfolio.dto';
@@ -19,15 +18,8 @@ export class PortfolioController {
@Get('all/portfolio')
@ApiOkResponse({ type: PortfolioResourceDto })
all(
@Req() request: Request & { user: AccessPayload },
@Query() query: InvestmentReadQueryDto,
) {
return this.portfolio.read(
request.user.sub,
'all',
query.sync === 'observe',
);
all(@Req() request: Request & { user: AccessPayload }) {
return this.portfolio.read(request.user.sub, 'all');
}
@Get(':accountId/portfolio')
@@ -35,12 +27,7 @@ export class PortfolioController {
get(
@Req() request: Request & { user: AccessPayload },
@Param('accountId') accountId: string,
@Query() query: InvestmentReadQueryDto,
) {
return this.portfolio.read(
request.user.sub,
accountId,
query.sync === 'observe',
);
return this.portfolio.read(request.user.sub, accountId);
}
}
@@ -2,11 +2,12 @@ import { Module } from '@nestjs/common';
import { TbankModule } from '../../../integrations/tbank/tbank.module';
import { AuthModule } from '../../auth/auth.module';
import { TbankSyncModule } from '../tbank-sync/tbank-sync.module';
import { TbankConnectionController } from './tbank-connection.controller';
import { TbankConnectionService } from './tbank-connection.service';
@Module({
imports: [AuthModule, TbankModule],
imports: [AuthModule, TbankModule, TbankSyncModule],
controllers: [TbankConnectionController],
providers: [TbankConnectionService],
})
@@ -35,11 +35,22 @@ export class TbankConnectionService {
this.tbank.validateToken(token),
);
const encryptedToken = this.tokens.encrypt(token);
const confirmed = supportedTbankAccounts(accounts).map(({ id }) => id);
const connection = await this.prisma.$transaction(async (tx) => {
// Cascade old private data and attempts in the same transaction that
// publishes the validated membership and the new connection revision.
await tx.tbankConnection.deleteMany({ where: { userId } });
const now = new Date();
// Value history survives token replacement but is visible only for
// accounts confirmed by the new token.
await tx.tbankPortfolioValueHistory.updateMany({
where: { userId, accountId: { in: confirmed } },
data: { deletedAt: null },
});
await tx.tbankPortfolioValueHistory.updateMany({
where: { userId, accountId: { notIn: confirmed }, deletedAt: null },
data: { deletedAt: now },
});
return tx.tbankConnection.create({
data: {
userId,
@@ -66,6 +77,14 @@ export class TbankConnectionService {
}
async delete(userId: string): Promise<void> {
await this.prisma.tbankConnection.deleteMany({ where: { userId } });
await this.prisma.$transaction(async (tx) => {
await tx.tbankConnection.deleteMany({ where: { userId } });
// History stays stored but hidden until a new connection reconfirms
// the same account.
await tx.tbankPortfolioValueHistory.updateMany({
where: { userId, deletedAt: null },
data: { deletedAt: new Date() },
});
});
}
}
@@ -0,0 +1,226 @@
import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
import { randomUUID } from 'node:crypto';
import { TbankErrorCode } from '../../../integrations/tbank/tbank-error-code';
import {
TbankAccessService,
TbankMembershipService,
type TbankSyncAttempt,
} from '../../../integrations/tbank/tbank.api';
import { PrismaService } from '../../../prisma/prisma.service';
import { BrokerageOperationsSyncService } from '../operations/operations.api';
import { PortfolioService } from '../portfolio/portfolio.api';
import { UpcomingEventsService } from '../upcoming-events/upcoming-events.service';
import {
type HourlySection,
TbankSyncRepository,
} from './tbank-sync.repository';
const HOUR_MS = 3_600_000;
const TICK_MS = 60_000;
const WORK_POLL_MS = 100;
const FIRST_DAY_OF_MONTH = 1;
const ONE_ATTEMPT = 1;
const NO_ATTEMPTS = 0;
type SettledWork = { errorCode: null | string; id: string; state: string };
// One native timer per process. The timer only proposes work; the durable
// reservation in TbankSyncRepository is the fence that keeps a single
// attempt per connection. The deployment runs a single backend process.
// ponytail: no inter-process lease; add one before scaling past one process.
@Injectable()
export class TbankHourlyScheduler implements OnModuleDestroy, OnModuleInit {
private readonly processRunId = randomUUID();
private timer: NodeJS.Timeout | null = null;
private ticking = false;
constructor(
private readonly prisma: PrismaService,
private readonly repository: TbankSyncRepository,
private readonly access: TbankAccessService,
private readonly membership: TbankMembershipService,
private readonly portfolio: PortfolioService,
private readonly operations: BrokerageOperationsSyncService,
private readonly events: UpcomingEventsService,
) {}
onModuleInit() {
this.timer = setInterval(() => void this.tick(), TICK_MS);
}
onModuleDestroy() {
if (this.timer) clearInterval(this.timer);
}
async tick() {
if (this.ticking) return;
this.ticking = true;
try {
const connections = await this.prisma.tbankConnection.findMany({
where: { invalidToken: false },
select: { userId: true },
});
for (const { userId } of connections) {
await this.launch(userId);
}
} catch {
// A failed tick only delays the proposal until the next tick.
} finally {
this.ticking = false;
}
}
// A fresh connection starts its first walk immediately, regardless of the
// previous term of the replaced connection.
private async launch(userId: string) {
const reserved = await this.repository.reserveHourly(
userId,
this.processRunId,
HOUR_MS,
);
if (!reserved) return null;
const { userId: owner, revision, id, attemptId } = reserved.work;
setImmediate(() => {
void this.walk({ userId: owner, revision, id, attemptId });
});
return reserved.work;
}
// One membership pass, then the owners refresh their own datasets. The
// walk publishes only safe section results and holds no transaction or
// upstream call inside one.
private async walk(attempt: TbankSyncAttempt) {
const failures: HourlySection[] = [];
const attempted = { count: NO_ATTEMPTS };
try {
const { userId, revision } = attempt;
await this.refreshMembership(userId, revision, failures, attempted);
if (failures.some((item) => item.errorCode === TbankErrorCode.InvalidToken)) {
return void this.settle(attempt, failures, attempted.count);
}
const current = await this.access.local(userId);
if (current.revision !== revision) {
return void this.settle(attempt, failures, attempted.count);
}
await this.refreshAccounts(userId, revision, current.accounts, failures, attempted);
await this.settle(attempt, failures, attempted.count);
} catch {
await this.settle(attempt, failures, attempted.count);
}
}
private async refreshMembership(
userId: string,
revision: string,
failures: HourlySection[],
attempted: { count: number },
) {
const member = await this.membership.request(userId, revision, true);
attempted.count += ONE_ATTEMPT;
if (!member) return;
await member.done;
const errorCode = await this.repository.workError(userId, member.id);
if (errorCode) failures.push({ section: 'membership', errorCode });
}
private async refreshAccounts(
userId: string,
revision: string,
accounts: Awaited<ReturnType<TbankAccessService['local']>>['accounts'],
failures: HourlySection[],
attempted: { count: number },
) {
for (const account of accounts) {
await this.refreshPortfolio(userId, revision, account.accountId, failures, attempted);
}
for (const account of accounts) {
await this.refreshOperations(userId, revision, account.accountId, failures, attempted);
}
for (const account of accounts) {
await this.refreshEvents(userId, account.accountId, failures, attempted);
}
}
private async refreshPortfolio(userId: string, revision: string, accountId: string, failures: HourlySection[], attempted: { count: number }) {
const work = await this.portfolio.request(userId, revision, accountId, true);
attempted.count += ONE_ATTEMPT;
if (!work) return;
await work.done;
const errorCode = await this.repository.workError(userId, work.id);
if (errorCode) failures.push({ section: 'portfolio', errorCode, accountId });
}
private async refreshOperations(userId: string, revision: string, accountId: string, failures: HourlySection[], attempted: { count: number }) {
attempted.count += ONE_ATTEMPT;
try {
const id = await this.operations.request(userId, [{ accountId, from: monthStartOf(new Date()), to: new Date() }], true, 'screen', revision);
if (!id) return;
const errorCode = await this.settledWorkError(userId, [id]);
if (errorCode) failures.push({ section: 'operations', errorCode, accountId });
} catch {
failures.push({ section: 'operations', errorCode: TbankErrorCode.Unavailable, accountId });
}
}
private async refreshEvents(userId: string, accountId: string, failures: HourlySection[], attempted: { count: number }) {
attempted.count += ONE_ATTEMPT;
try {
const ids = await this.events.refresh(userId, accountId);
const failed = ids.length ? (await this.settledWorks(userId, ids)).find((row) => row.state === 'failed') : undefined;
if (failed?.errorCode) failures.push({ section: 'events', errorCode: failed.errorCode, accountId });
else await this.events.get(userId, accountId);
} catch {
failures.push({ section: 'events', errorCode: TbankErrorCode.Unavailable, accountId });
}
}
private async settle(
attempt: TbankSyncAttempt,
failures: HourlySection[],
attempted: number,
) {
let result: 'failed' | 'partial' | 'succeeded' = 'succeeded';
if (failures.length) {
result = attempted > NO_ATTEMPTS && failures.length >= attempted ? 'failed' : 'partial';
}
await this.repository
.settleHourly(attempt, result, failures)
.catch(() => undefined);
}
private async settledWorkError(userId: string, ids: string[]) {
const rows = await this.settledWorks(userId, ids);
return rows.find((row) => row.state === 'failed')?.errorCode ?? null;
}
// Polls the durable work rows until they leave the running state; a row
// deleted with its connection counts as interrupted, not failed.
private async settledWorks(userId: string, ids: string[]) {
const settled = new Map<string, SettledWork>();
const pending = new Set(ids);
while (pending.size) {
const rows = await this.repository.works(userId, [...pending]);
for (const row of rows) {
if (row.state !== 'running') {
settled.set(row.id, row);
pending.delete(row.id);
}
}
for (const id of [...pending]) {
if (!rows.some((row) => row.id === id)) {
settled.set(id, { id, state: 'cancelled', errorCode: null });
pending.delete(id);
}
}
if (pending.size) await new Promise((resolve) => setTimeout(resolve, WORK_POLL_MS));
}
return [...settled.values()];
}
}
function monthStartOf(now: Date) {
return new Date(
Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), FIRST_DAY_OF_MONTH),
);
}
@@ -1,22 +1,8 @@
import type { Request } from 'express';
import { Controller, Get, Param, Req, UseGuards } from '@nestjs/common';
import {
BadRequestException,
Body,
Controller,
Get,
HttpCode,
HttpStatus,
Param,
Post,
Req,
UseGuards,
ValidationPipe,
} from '@nestjs/common';
import {
ApiAcceptedResponse,
ApiBearerAuth,
ApiBody,
ApiExtraModels,
ApiOkResponse,
ApiTags,
@@ -27,9 +13,6 @@ import { AccessPayload, JwtAuthGuard } from '../../auth/auth.api';
import {
EventsInvestmentScreenDto,
IncomeInvestmentScreenDto,
InvestmentScreenDto,
investmentScreenSchema,
TbankSyncAcceptedDto,
TbankSyncStatusDto,
TimestampInvestmentScreenDto,
} from './tbank-sync.dto';
@@ -37,9 +20,10 @@ import { TbankSyncService } from './tbank-sync.service';
@ApiTags('tbank-sync')
@ApiExtraModels(
IncomeInvestmentScreenDto,
EventsInvestmentScreenDto,
IncomeInvestmentScreenDto,
TimestampInvestmentScreenDto,
TbankSyncStatusDto,
)
@ApiProblemResponses()
@ApiBearerAuth()
@@ -47,28 +31,6 @@ import { TbankSyncService } from './tbank-sync.service';
@Controller('tbank-sync')
export class TbankSyncController {
constructor(private readonly sync: TbankSyncService) {}
@Post()
@HttpCode(HttpStatus.ACCEPTED)
@ApiBody({ schema: investmentScreenSchema })
@ApiAcceptedResponse({ type: TbankSyncAcceptedDto })
request(
@Req() request: Request & { user: AccessPayload },
@Body(
new ValidationPipe({
forbidUnknownValues: true,
expectedType: InvestmentScreenDto,
transform: true,
whitelist: true,
forbidNonWhitelisted: true,
validationError: { target: false, value: false },
exceptionFactory: (message) => new BadRequestException({ message }),
}),
)
input: unknown,
) {
return this.sync.request(request.user.sub, input as InvestmentScreenDto);
}
@Get(':id')
@ApiOkResponse({ type: TbankSyncStatusDto })
status(
@@ -227,3 +227,46 @@ export class TbankSyncStatusDto {
@ApiProperty({ type: String, nullable: true }) declare errorCode:
null | string;
}
export class HourlySyncSectionDto {
@ApiProperty({ enum: ['membership', 'portfolio', 'operations', 'events'] })
declare section: 'events' | 'membership' | 'operations' | 'portfolio';
@ApiPropertyOptional({
type: String,
nullable: false,
description: 'Account identifier of the problematic section, when scoped.',
})
declare accountId?: string;
@ApiProperty({
type: String,
nullable: true,
description:
'Safe error code of the failed section. No tokens or upstream payloads.',
})
declare errorCode: null | string;
}
export class HourlySyncStatusDto {
@ApiProperty({
enum: ['running', 'succeeded', 'partial', 'failed', 'cancelled'],
description:
'Last hourly walk state. Cancelled means an interrupted attempt; its start still delays the next term.',
})
declare state:
| 'cancelled' |
'failed' |
'partial' |
'running' |
'succeeded';
@ApiProperty({ format: 'date-time' })
declare startedAt: string;
@ApiProperty({ type: String, nullable: true, format: 'date-time' })
declare finishedAt: null | string;
@ApiProperty({ type: HourlySyncSectionDto, isArray: true })
declare sections: HourlySyncSectionDto[];
}
@@ -5,6 +5,8 @@ import { AuthModule } from '../../auth/auth.module';
import { InstrumentSourcesModule } from '../instruments/instrument-sources.module';
import { OperationsModule } from '../operations/operations.module';
import { PortfolioModule } from '../portfolio/portfolio.module';
import { UpcomingEventsModule } from '../upcoming-events/upcoming-events.module';
import { TbankHourlyScheduler } from './tbank-hourly.scheduler';
import { TbankSyncController } from './tbank-sync.controller';
import { TbankSyncRepository } from './tbank-sync.repository';
import { TbankSyncService } from './tbank-sync.service';
@@ -15,8 +17,10 @@ import { TbankSyncService } from './tbank-sync.service';
PortfolioModule,
OperationsModule,
InstrumentSourcesModule,
UpcomingEventsModule,
],
controllers: [TbankSyncController],
providers: [TbankSyncService, TbankSyncRepository],
providers: [TbankSyncService, TbankSyncRepository, TbankHourlyScheduler],
exports: [TbankHourlyScheduler, TbankSyncRepository],
})
export class TbankSyncModule {}
@@ -12,6 +12,15 @@ export type ScreenProgress = {
children: string[];
dispatched: boolean;
};
export type HourlySection = {
accountId?: string;
errorCode: null | string;
section: 'events' | 'membership' | 'operations' | 'portfolio';
};
export type HourlyProgress = {
result?: 'failed' | 'partial' | 'succeeded';
sections: HourlySection[];
};
@Injectable()
export class TbankSyncRepository {
constructor(private readonly prisma: PrismaService) {}
@@ -123,4 +132,122 @@ export class TbankSyncRepository {
});
});
}
// The timer only proposes work; this reservation is the durable fence. The
// attempt start (createdAt) is the basis of the next hourly term, including
// failed and interrupted attempts.
reserveHourly(userId: string, runId: string, hourMs: number) {
return this.prisma.$transaction(async (tx) => {
const connection = await tx.tbankConnection.findFirst({
where: { userId, invalidToken: false },
select: { revision: true },
});
if (!connection) return null;
const key = {
userId,
revision: connection.revision,
scopeKey: 'hourly',
};
const previous = await tx.tbankSyncWork.findUnique({
where: { userId_revision_scopeKey: key },
});
const now = new Date();
if (previous) {
const due = now.valueOf() - previous.createdAt.valueOf() >= hourMs;
if (previous.state === 'running') {
if (previous.processRunId === runId) return null;
// A dead process cannot finish its attempt; interrupt it, but its
// start stays the basis of the next term.
await tx.tbankSyncWork.update({
where: { userId_revision_scopeKey: key },
data: { state: 'cancelled', errorCode: null },
});
if (!due) return null;
} else if (!due) return null;
}
const data = {
attemptId: randomUUID(),
processRunId: runId,
state: 'running',
priority: 0,
intent: JSON.parse(
JSON.stringify({ source: 'hourly' }),
) as Prisma.InputJsonObject,
progress: { sections: [] } as unknown as Prisma.InputJsonValue,
errorCode: null,
};
const work = await tx.tbankSyncWork.upsert({
where: { userId_revision_scopeKey: key },
create: { ...key, ...data, id: randomUUID() },
update: { ...data, createdAt: now },
});
return { work, launch: true };
});
}
settleHourly(
attempt: TbankSyncAttempt,
result: 'failed' | 'partial' | 'succeeded',
sections: HourlySection[],
) {
return this.prisma.tbankSyncWork.updateMany({
where: {
id: attempt.id,
attemptId: attempt.attemptId,
userId: attempt.userId,
revision: attempt.revision,
state: 'running',
},
data: {
state: result === 'failed' ? 'failed' : 'complete',
errorCode:
sections.find((section) => section.errorCode)?.errorCode ?? null,
progress: { result, sections },
},
});
}
async hourly(userId: string) {
const connection = await this.prisma.tbankConnection.findFirst({
where: { userId },
select: { revision: true, invalidToken: true },
});
if (!connection || connection.invalidToken) return null;
const work = await this.prisma.tbankSyncWork.findUnique({
where: {
userId_revision_scopeKey: {
userId,
revision: connection.revision,
scopeKey: 'hourly',
},
},
});
if (!work) return null;
const progress = work.progress as HourlyProgress;
let state: 'cancelled' | 'running' | HourlyProgress['result'];
if (work.state === 'running') state = 'running';
else if (work.state === 'cancelled') state = 'cancelled';
else state = progress.result ?? 'failed';
return {
startedAt: work.createdAt,
finishedAt: work.state === 'running' ? null : work.updatedAt,
state,
sections: progress.sections ?? [],
};
}
works(userId: string, ids: string[]) {
return this.prisma.tbankSyncWork.findMany({
where: { userId, id: { in: ids } },
select: { id: true, state: true, errorCode: true },
});
}
async workError(userId: string, id: string) {
const work = await this.prisma.tbankSyncWork.findFirst({
where: { userId, id },
select: { state: true, errorCode: true },
});
return work?.state === 'failed' ? work.errorCode : null;
}
}
@@ -119,7 +119,6 @@ export class TbankSyncService {
scope.screen === 'stock' || scope.screen === 'bond'
? 'all'
: accountScope,
true,
);
const positions = saved.data?.positions ?? [];
const instruments = [
@@ -45,64 +45,24 @@ export class UpcomingEventsService {
private readonly metadata: InstrumentMetadataService,
) {}
// Read endpoints stay local; callers without a saved portfolio receive unknown.
async get(
userId: string,
accountId: string,
query: UpcomingEventsQueryDto = {},
) {
): Promise<Awaited<ReturnType<ForecastService['result']>>> {
const window = dateWindow(query);
const connection = await this.access.local(userId, accountId);
const portfolio = await this.portfolios.read(
userId,
accountId,
query.sync === 'observe',
);
const sources: Awaited<ReturnType<CatalogEnrichmentService['readMany']>> =
[];
const portfolio = await this.portfolios.read(userId, accountId);
const events: UpcomingEventDto[] = [];
let ready = portfolio.data !== null;
const positions = portfolio.data?.positions.filter(supported) ?? [];
const metadata = await this.metadata.savedMany(positions);
for (const type of ['share', 'bond'] as const) {
const identities = positions
.filter(
(position) => position.type === (type === 'share' ? 'STOCK' : 'BOND'),
)
.map((position) => ({
uid:
position.instrumentUid ||
metadata.get(metadataKey(position))?.data?.uid,
ticker:
position.ticker ||
metadata.get(metadataKey(position))?.data?.ticker ||
'',
}))
.filter((entry): entry is { ticker: string; uid: string } =>
Boolean(entry.uid),
);
const uids = [...new Set(identities.map((entry) => entry.uid))];
for (
let offset = FIRST_BATCH_OFFSET;
offset < uids.length;
offset += calendarReadBatchSize
) {
sources.push(
...(await this.sources.readMany(
userId,
type,
uids.slice(offset, offset + calendarReadBatchSize),
query.sync === 'observe',
['detail', type === 'share' ? 'dividends' : 'bond-events'],
{
...window,
tickers: Object.fromEntries(
identities.map((entry) => [entry.uid, entry.ticker]),
),
},
)),
);
}
}
const { sources, metadata } = await this.readSources(
userId,
positions,
window,
query.sync === 'observe',
);
for (const position of positions) {
const generic = metadata.get(metadataKey(position))?.data;
const uid = position.instrumentUid || generic?.uid;
@@ -205,6 +165,85 @@ export class UpcomingEventsService {
getAll(userId: string, query: UpcomingEventsQueryDto = {}) {
return this.get(userId, 'all', query);
}
// Triggers online refreshes of the account's event sources and returns the
// launched work ids for the hourly walk to observe.
async refresh(userId: string, accountId: string): Promise<string[]> {
const window = dateWindow({});
await this.access.local(userId, accountId);
const portfolio = await this.portfolios.read(userId, accountId);
const positions = portfolio.data?.positions.filter(supported) ?? [];
const { launched } = await this.readSources(
userId,
positions,
window,
false,
);
return launched;
}
private async readSources(
userId: string,
positions: Position[],
window: { from: string; to: string },
observe: boolean,
) {
const metadata = await this.metadata.savedMany(positions);
const sources: Awaited<ReturnType<CatalogEnrichmentService['readMany']>> =
[];
const launched: string[] = [];
for (const type of ['share', 'bond'] as const) {
const identities = this.positionIdentities(positions, metadata, type);
const uids = [...new Set(identities.map((entry) => entry.uid))];
for (
let offset = FIRST_BATCH_OFFSET;
offset < uids.length;
offset += calendarReadBatchSize
) {
const batch = await this.sources.readMany(
userId,
type,
uids.slice(offset, offset + calendarReadBatchSize),
observe,
['detail', type === 'share' ? 'dividends' : 'bond-events'],
{
...window,
tickers: Object.fromEntries(
identities.map((entry) => [entry.uid, entry.ticker]),
),
},
);
sources.push(...batch);
for (const source of batch) {
if (source.meta.sync.state === 'running' && source.meta.sync.id) {
launched.push(source.meta.sync.id);
}
}
}
}
return { sources, metadata, launched };
}
private positionIdentities(
positions: Position[],
metadata: Awaited<ReturnType<InstrumentMetadataService['savedMany']>>,
type: 'bond' | 'share',
) {
return positions
.filter((position) => position.type === (type === 'share' ? 'STOCK' : 'BOND'))
.map((position) => ({
uid:
position.instrumentUid ||
metadata.get(metadataKey(position))?.data?.uid,
ticker:
position.ticker ||
metadata.get(metadataKey(position))?.data?.ticker ||
'',
}))
.filter((entry): entry is { ticker: string; uid: string } =>
Boolean(entry.uid),
);
}
}
function supported(position: Position): boolean {