WealthLedger/backend
5
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases Wiki Activity

feat!: persist T-Bank data and synchronize reads
Backend security / security (push) Successful in 8s
Details
Backend verify / verify (push) Failing after 10m45s
Details
Conventional commits / commits (pull_request) Successful in 29s
Details
Backend security / security (pull_request) Successful in 8s
Details
Backend verify / verify (pull_request) Failing after 10m37s
Details

Browse Source 
Store account membership, portfolio, operations, forecasts and market sources in SQLite.
Coalesce scoped refreshes, fence connection revisions, preserve complete financial
snapshots and stabilize operation period state.

BREAKING CHANGE: investment reads return data/meta resource envelopes and locally
persisted snapshots; consumers must use the matching OpenAPI contract and explicit
synchronization endpoints.
This commit is contained in:
ksv741
2026-09-26 21:29:15 +03:00
parent 0862d56c4e
commit d6e6c4ace5
116 changed files with 18321 additions and 5409 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/modules/investments/portfolio/application/portfolio.service.ts
+365 -131
View File
@@ -1,4 +1,17 @@
import { Injectable, ServiceUnavailableException } from '@nestjs/common';
import { CandleService } from '../../instruments/instruments.api';
import { randomUUID } from 'node:crypto';
import { StructuredLogger } from '../../../../common/logging/structured-logger.service';
import { investmentResourceMeta } from '../../investment-resource.mapper';
import {
PortfolioRepository,
portfolioFreshForMs,
portfolioWorkKey,
} from '../infrastructure/portfolio.repository';
import {
Injectable,
ServiceUnavailableException,
NotFoundException,
} from '@nestjs/common';
import {
add,
formatDecimal,
@@ -10,10 +23,14 @@ import { RequestContextService } from '../../../../common/logging/request-contex
import { MoneyDto } from '../../../../common/money.dto';
import {
InstrumentMetadataService,
metadataKey,
type TbankInstrument,
type MetadataIdentity,
TbankAccessService,
TbankClient,
TbankConnectionContext,
TbankMoney,
TbankMembershipService,
tbankSourceTtlMs,
} from '../../../../integrations/tbank/tbank.api';
import { decimal } from '../../../../integrations/tbank/decimal';
import { portfolioStructure } from '../domain/portfolio-structure';
@@ -24,10 +41,7 @@ import {
PortfolioPositionDto,
} from '../presentation/portfolio.dto';
const freshForMs = 30_000;
const fallbackForMs = 300_000;
const positionsBatchSize = 5;
const metadataTimeoutMs = 30_000;
const membershipFreshForMs = tbankSourceTtlMs.membership;
const types: Record<string, string> = {
share: 'STOCK',
bond: 'BOND',
@@ -39,116 +53,358 @@ const types: Record<string, string> = {
dfa: 'DIGITAL_FINANCIAL_ASSET',
};
type CachedSnapshot = {
generation: string;
snapshot: PortfolioDto;
storedAt: number;
};
@Injectable()
export class PortfolioService {
private readonly snapshots = new Map<string, CachedSnapshot>();
private readonly processRunId = randomUUID();
private readonly active = new Set<string>();
private readonly pending = new Map<string, Promise<void>>();
private reservations: Promise<unknown> = Promise.resolve();
constructor(
private readonly access: TbankAccessService,
private readonly tbank: TbankClient,
private readonly requestContext: RequestContextService,
private readonly metadata: InstrumentMetadataService,
private readonly repository: PortfolioRepository,
private readonly membership: TbankMembershipService,
private readonly logger: StructuredLogger,
private readonly candles: CandleService,
) {}
async get(userId: string, accountId: string): Promise<PortfolioDto> {
async saved(userId: string, accountId: string, observe = false) {
this.requestContext.set({ userId, brokerageAccountId: accountId });
return this.access.withConnection(userId, (connection) =>
this.snapshot(userId, accountId, connection),
);
}
async getAll(userId: string): Promise<PortfolioDto> {
return this.access.withSupportedAccounts(userId, async (context) => {
const portfolios: PortfolioDto[] = [];
for (const account of context.accounts) {
const portfolio = await this.snapshot(userId, account.id, context);
portfolios.push({
...portfolio,
positions: portfolio.positions.map((position) => ({
...position,
account: { id: account.id, name: account.name },
})),
});
}
return {
lastSuccessfulAt:
portfolios.map((portfolio) => portfolio.lastSuccessfulAt).sort()[0] ??
new Date().toISOString(),
isStale: portfolios.some((portfolio) => portfolio.isStale),
totalsByCurrency: sumMoneyByCurrency(
portfolios.flatMap((portfolio) => portfolio.totalsByCurrency),
),
cashBalances: sumMoneyByCurrency(
portfolios.flatMap((portfolio) => portfolio.cashBalances),
),
positions: portfolios.flatMap((portfolio) => portfolio.positions),
structure: portfolioStructure(
portfolios.flatMap((portfolio) => portfolio.positions),
portfolios.flatMap((portfolio) => portfolio.cashBalances),
),
};
});
}
private async snapshot(
userId: string,
accountId: string,
{ token, generation }: TbankConnectionContext,
): Promise<PortfolioDto> {
const key = `${userId}:${accountId}`;
const cached = this.snapshots.get(key);
if (
cached?.generation === generation &&
Date.now() - cached.storedAt <= freshForMs
)
return cached.snapshot;
try {
const snapshot = await this.access.withAccountForConnection(
{ token, generation },
accountId,
async () => {
const [portfolio, positions] = await Promise.all([
this.tbank.getPortfolio(token, accountId),
this.tbank.getPositions(token, accountId),
]);
return this.normalize(
token,
portfolio.positions ?? [],
positions.money ?? [],
);
},
const connection = await this.access.local(userId, accountId);
if (!observe) {
const discovery = await this.membership.request(
userId,
connection.revision,
);
this.snapshots.set(key, {
generation,
snapshot,
storedAt: Date.parse(snapshot.lastSuccessfulAt),
});
return snapshot;
} catch (error) {
if (
error instanceof ServiceUnavailableException &&
cached?.generation === generation &&
Date.now() - cached.storedAt <= fallbackForMs
) {
return { ...cached.snapshot, isStale: true };
}
throw error;
if (discovery)
void discovery.done
.then(async () => {
const current = await this.access.local(userId, accountId);
if (current.revision !== connection.revision) return;
for (const account of current.accounts.filter(
(account) =>
accountId === 'all' || account.accountId === accountId,
))
await this.queueSnapshot(
userId,
current.revision,
account.accountId,
);
})
.catch(() =>
this.logger.event(
'warn',
'tbank.portfolio.discovery_not_available',
{ userId },
),
);
for (const account of (discovery ? [] : connection.accounts).filter(
(account) => accountId === 'all' || account.accountId === accountId,
))
await this.queueSnapshot(
userId,
connection.revision,
account.accountId,
);
}
const saved = await this.repository.read(userId, connection.revision);
if (
saved?.membershipAt &&
accountId !== 'all' &&
!saved.accounts.some((account) => account.accountId === accountId)
)
throw new NotFoundException();
if (!observe) {
const identities = new Map<string, MetadataIdentity>();
for (const account of (saved?.accounts ?? []).filter(
(account) => accountId === 'all' || account.accountId === accountId,
)) {
const snapshot = account.portfolio?.payload as unknown as
PortfolioDto | undefined;
for (const position of snapshot?.positions ?? [])
identities.set(metadataKey(position), position);
}
const savedMetadata = await this.metadata.savedMany([
...identities.values(),
]);
for (const identity of identities.values())
if (!savedMetadata.get(metadataKey(identity))?.fresh)
await this.metadata.request(userId, connection.revision, identity);
}
if (!observe)
void this.candles
.prefetch(
userId,
connection.revision,
(saved?.accounts ?? [])
.filter(
(account) =>
accountId === 'all' || account.accountId === accountId,
)
.flatMap(
(account) =>
(
account.portfolio?.payload as unknown as
PortfolioDto | undefined
)?.positions ?? [],
),
)
.catch(() =>
this.logger.event('warn', 'portfolio.candle-prefetch-unavailable'),
);
const membershipMeta = investmentResourceMeta({
resourceKey: 'accounts/membership',
dataRevision: String(saved?.membershipRevision ?? 0),
availability: saved?.membershipAt ? 'complete' : 'missing',
fetchedAt: saved?.membershipAt ?? null,
freshForMs: membershipFreshForMs,
work: saved?.work.find((work) => work.scopeKey === 'membership') ?? null,
});
return {
membership: membershipMeta,
accounts: (saved?.accounts ?? [])
.filter(
(account) => accountId === 'all' || account.accountId === accountId,
)
.map((account) => ({
id: account.accountId,
name: account.name,
type: account.type,
openedOn: account.openedOn,
snapshot: account.portfolio
? {
...(account.portfolio.payload as unknown as PortfolioDto),
lastSuccessfulAt: account.portfolio.fetchedAt.toISOString(),
isStale:
Date.now() - account.portfolio.fetchedAt.valueOf() >=
portfolioFreshForMs,
}
: null,
meta: investmentResourceMeta({
resourceKey: `portfolio/${encodeURIComponent(account.accountId)}`,
dataRevision: String(account.portfolio?.dataRevision ?? 0),
availability: account.portfolio ? 'complete' : 'missing',
fetchedAt: account.portfolio?.fetchedAt ?? null,
freshForMs: portfolioFreshForMs,
work:
saved?.work.find(
(work) => work.scopeKey === portfolioWorkKey(account.accountId),
) ?? null,
}),
})),
};
}
invalidate(userId: string): void {
for (const key of this.snapshots.keys())
if (key.startsWith(`${userId}:`)) this.snapshots.delete(key);
async read(userId: string, accountId: string, observe = false) {
const saved = await this.saved(userId, accountId, observe);
const missing =
saved.membership.availability === 'missing' ||
saved.accounts.some((account) => !account.snapshot);
const metadata = await this.metadata.savedMany(
saved.accounts.flatMap((account) => account.snapshot?.positions ?? []),
);
const metadataDependencies = [...metadata.entries()].flatMap(
([key, source]) =>
source.complete && source.data
? [
investmentResourceMeta({
resourceKey: `instrument/metadata/${key}`,
dataRevision: source.dataRevision,
availability: 'complete',
fetchedAt: source.fetchedAt,
freshForMs: tbankSourceTtlMs.metadata,
work: null,
}),
]
: [],
);
const portfolios = saved.accounts.flatMap((account) =>
account.snapshot
? [
{
...account.snapshot,
positions: account.snapshot.positions.map((position) => {
const shared = metadata.get(metadataKey(position))?.data;
return {
...position,
...(shared
? {
displayName:
shared.name?.trim() ||
shared.ticker?.trim() ||
position.displayName,
ticker: shared.ticker?.trim() || position.ticker,
isin: shared.isin?.trim() || position.isin,
logoName:
shared.brand?.logoName?.trim() || position.logoName,
}
: {}),
...(accountId === 'all'
? { account: { id: account.id, name: account.name } }
: {}),
};
}),
},
]
: [],
);
const lastSuccessfulAt =
portfolios.map((portfolio) => portfolio.lastSuccessfulAt).sort()[0] ??
saved.membership.lastSuccessfulAt;
const data =
missing || !lastSuccessfulAt
? null
: {
lastSuccessfulAt,
isStale: portfolios.length
? portfolios.some((portfolio) => portfolio.isStale)
: saved.membership.freshness !== 'fresh',
totalsByCurrency: sumMoneyByCurrency(
portfolios.flatMap((portfolio) => portfolio.totalsByCurrency),
),
cashBalances: sumMoneyByCurrency(
portfolios.flatMap((portfolio) => portfolio.cashBalances),
),
positions: portfolios.flatMap((portfolio) => portfolio.positions),
structure: portfolioStructure(
portfolios.flatMap((portfolio) => portfolio.positions),
portfolios.flatMap((portfolio) => portfolio.cashBalances),
),
};
const work =
saved.accounts
.map((account) => account.meta.sync)
.find((work) => work.state === 'running') ??
saved.accounts
.map((account) => account.meta.sync)
.find((work) => work.state === 'failed') ??
saved.membership.sync;
return {
data,
meta: investmentResourceMeta({
resourceKey: `portfolio/${encodeURIComponent(accountId)}`,
dataRevision: JSON.stringify([
saved.accounts.map((account) => [
account.id,
account.meta.dataRevision,
]),
metadataDependencies.map((dependency) => dependency.dataRevision),
]),
availability: missing
? portfolios.length
? 'partial'
: 'missing'
: 'complete',
fetchedAt: lastSuccessfulAt ? new Date(lastSuccessfulAt) : null,
freshForMs: portfolios.length
? portfolioFreshForMs
: membershipFreshForMs,
work: work.id ? { ...work, id: work.id } : null,
dependencies: [
saved.membership,
...saved.accounts.map((account) => account.meta),
...metadataDependencies,
],
}),
};
}
request(userId: string, revision: string, accountId: string, force = false) {
return this.queueSnapshot(userId, revision, accountId, force);
}
private queueSnapshot(
userId: string,
revision: string,
accountId: string,
force = false,
) {
const reserved = this.reservations.then(async () => {
const requested = await this.repository.reserve(
userId,
revision,
accountId,
this.processRunId,
this.active,
force,
);
if (!requested) return null;
if (!requested.launch)
return {
id: requested.work.id,
done: this.pending.get(requested.work.attemptId) ?? Promise.resolve(),
};
const { id, attemptId } = requested.work;
this.active.add(attemptId);
const done = new Promise<void>((resolve) => setImmediate(resolve))
.then(async () => {
const attempt = { id, attemptId, userId, revision };
try {
const token = await this.access.tokenForAttempt(attempt);
if (!token) return;
const [portfolio, positions] = await Promise.all([
this.tbank.getPortfolio(token, accountId),
this.tbank.getPositions(token, accountId),
]);
const snapshot = await this.normalize(
portfolio.positions ?? [],
positions.money ?? [],
);
const published = await this.repository.publish(
attempt,
accountId,
snapshot,
);
if (published) {
for (const position of snapshot.positions)
await this.metadata.request(userId, revision, position);
if (!force)
await this.candles.prefetch(
userId,
revision,
snapshot.positions,
);
}
} catch (error) {
if (
error instanceof Error &&
error.message === 'INVALID_TBANK_TOKEN'
)
await this.access.rejectMembershipToken(attempt);
else await this.repository.fail(attempt);
}
})
.catch(() =>
this.logger.event('error', 'tbank.portfolio.persistence_failed', {
userId,
}),
)
.finally(() => {
this.active.delete(attemptId);
this.pending.delete(attemptId);
});
this.pending.set(attemptId, done);
return { id, done };
});
this.reservations = reserved.catch(() => undefined);
return reserved;
}
async get(userId: string, accountId: string): Promise<PortfolioDto> {
const result = await this.read(userId, accountId);
if (!result.data)
throw new ServiceUnavailableException({ code: 'TBANK_UNAVAILABLE' });
return result.data;
}
getAll(userId: string): Promise<PortfolioDto> {
return this.get(userId, 'all');
}
private async normalize(
token: string,
upstream: {
figi?: string;
instrumentUid?: string;
@@ -160,28 +416,13 @@ export class PortfolioService {
money: TbankMoney[],
): Promise<PortfolioDto> {
const lastSuccessfulAt = new Date().toISOString().replace(/\.\d{3}Z$/, 'Z');
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), metadataTimeoutMs);
const positions: (PortfolioPositionDto | null)[] = [];
try {
for (
let offset = 0;
offset < upstream.length;
offset += positionsBatchSize
) {
positions.push(
...(await Promise.all(
upstream
.slice(offset, offset + positionsBatchSize)
.map((position) =>
this.position(token, position, controller.signal),
),
)),
);
}
} finally {
clearTimeout(timeout);
}
const savedMetadata = await this.metadata.savedMany(upstream);
const positions = upstream.map((position) =>
this.position(
position,
savedMetadata.get(metadataKey(position))?.data ?? null,
),
);
const cash = new Map<string, ReturnType<typeof decimal>>();
for (const value of money) {
const currency = value.currency?.trim().toUpperCase();
@@ -206,11 +447,10 @@ export class PortfolioService {
};
}
private async position(
token: string,
position: Parameters<PortfolioService['normalize']>[1][number],
signal: AbortSignal,
): Promise<PortfolioPositionDto | null> {
private position(
position: Parameters<PortfolioService['normalize']>[0][number],
metadata: TbankInstrument | null,
): PortfolioPositionDto | null {
const quantity = decimal(position.quantity ?? {});
if (isZero(quantity)) return null;
const figi = position.figi?.trim() || null;
@@ -235,12 +475,6 @@ export class PortfolioService {
),
}
: null;
const metadata = await this.metadata.resolve(
token,
instrumentUid,
figi,
signal,
);
return {
figi,
instrumentUid,