feat!: persist T-Bank data and synchronize reads

Store account membership, portfolio, operations, forecasts and market sources in SQLite. Coalesce scoped refreshes, fence connection revisions, preserve complete financial snapshots and stabilize operation period state. BREAKING CHANGE: investment reads return data/meta resource envelopes and locally persisted snapshots; consumers must use the matching OpenAPI contract and explicit synchronization endpoints.
2026-09-26 21:29:15 +03:00
parent 0862d56c4e
commit d6e6c4ace5
116 changed files with 18321 additions and 5409 deletions
@@ -0,0 +1,481 @@
+import { planInvestmentScreen } from '../../investment-sync-plan';
+import { StructuredLogger } from '../../../../common/logging/structured-logger.service';
+import {
+  Injectable,
+  NotFoundException,
+  ServiceUnavailableException,
+} from '@nestjs/common';
+import { randomUUID } from 'node:crypto';
+import { MILLISECONDS_PER_DAY } from '../../../../common/time';
+import { formatDecimal } from '../../../../common/decimal';
+import { decimal } from '../../../../integrations/tbank/decimal';
+import {
+  TbankAccessService,
+  TbankClient,
+  type TbankShare,
+} from '../../../../integrations/tbank/tbank.api';
+import type { TbankSyncAttempt } from '../../../../integrations/tbank/tbank-access.service';
+import { investmentResourceMeta } from '../../investment-resource.mapper';
+import { CatalogEnrichmentService } from './catalog-enrichment.service';
+import { CandleRepository } from '../infrastructure/candle.repository';
+import {
+  candleGaps,
+  candleWindows,
+  utcDay,
+  type CandleRange,
+} from '../domain/candle-coverage';
+import { timestamp } from './instrument-market';
+
+type CandleInput = {
+  uid: string;
+  instrumentType: 'share' | 'bond';
+  period: 'month' | 'halfYear' | 'year' | 'all';
+  range?: CandleRange;
+};
+type Producer = {
+  input: CandleInput;
+  attempt: TbankSyncAttempt;
+  now: Date;
+  force: boolean;
+  priority: number;
+  observers: TbankSyncAttempt[];
+  ranges?: CandleRange[];
+  nextWindow?: number;
+};
+const months = { month: 1, halfYear: 6, year: 12 };
+const tailFreshForMs = 60_000;
+const prefetchReadBatchSize = 100;
+function scope(input: CandleInput) {
+  return `candles:${input.uid}:${input.period}${input.range ? `:${input.range.from.toISOString()}/${input.range.to.toISOString()}` : ''}`;
+}
+function rangeFor(
+  input: CandleInput,
+  instrument: TbankShare,
+  now: Date,
+): CandleRange {
+  if (input.range) return input.range;
+  const from = utcDay(now);
+  const history = timestamp(instrument.first_1dayCandleDate);
+  if (input.period === 'all') {
+    if (!history)
+      throw new ServiceUnavailableException({
+        code:
+          input.instrumentType === 'share'
+            ? 'STOCK_HISTORY_START_UNAVAILABLE'
+            : 'BOND_HISTORY_START_UNAVAILABLE',
+      });
+    from.setTime(utcDay(new Date(history)).valueOf());
+  } else {
+    from.setUTCMonth(from.getUTCMonth() - months[input.period]);
+    if (history && Date.parse(history) > from.valueOf())
+      from.setTime(utcDay(new Date(history)).valueOf());
+  }
+  return { from, to: new Date(utcDay(now).valueOf() + MILLISECONDS_PER_DAY) };
+}
+@Injectable()
+export class CandleService {
+  private readonly runId = randomUUID();
+  private readonly active = new Set<string>();
+  private readonly producers = new Map<string, Producer>();
+  private readonly queue: Producer[] = [];
+  private reservations: Promise<unknown> = Promise.resolve();
+  private running = false;
+  private readonly prefetches = new Map<string, Promise<void>>();
+  constructor(
+    private readonly repository: CandleRepository,
+    private readonly sources: CatalogEnrichmentService,
+    private readonly access: TbankAccessService,
+    private readonly tbank: TbankClient,
+    private readonly logger: StructuredLogger,
+  ) {}
+
+  async read(userId: string, input: CandleInput, observe = false) {
+    const { revision } = await this.access.local(userId);
+    const [detail] = await this.sources.readMany(
+      userId,
+      input.instrumentType,
+      [input.uid],
+      true,
+      ['detail'],
+    );
+    if (detail.meta.availability === 'complete' && !detail.instrument)
+      throw new NotFoundException();
+    const now = new Date();
+    const range = detail.instrument
+      ? rangeFor(input, detail.instrument, now)
+      : null;
+    const saved = range ? await this.repository.read(input.uid, range) : null;
+    const gaps = range && saved ? candleGaps(range, saved.coverage, now) : [];
+    if (!observe && (!saved || gaps.length))
+      await this.request(userId, revision, input, now);
+    const work = await this.repository.work(userId, revision, scope(input));
+    const fetchedAt =
+      saved?.coverage
+        .map((row) => row.fetchedAt)
+        .sort((a, b) => a.valueOf() - b.valueOf())[0] ?? null;
+    const covered = saved?.coverage.length ? true : false;
+    const completeCoverage = (saved?.coverage ?? [])
+      .filter((row) => row.isComplete || row.from >= utcDay(now))
+      .map((row) => ({ ...row, isComplete: true }));
+    const missing = range ? candleGaps(range, completeCoverage, now) : [];
+    const complete = Boolean(saved && range && !missing.length);
+    const publicRange = (value: CandleRange) => ({
+      from: value.from.toISOString(),
+      to: value.to.toISOString(),
+    });
+    return {
+      data:
+        !saved || !covered
+          ? null
+          : {
+              candles: saved.data,
+              currency:
+                input.instrumentType === 'share'
+                  ? detail.instrument?.currency?.toUpperCase() || null
+                  : null,
+              historyFrom: timestamp(detail?.instrument?.first_1dayCandleDate),
+              observedAt: fetchedAt?.toISOString() ?? now.toISOString(),
+            },
+      meta: investmentResourceMeta({
+        resourceKey: scope(input),
+        dataRevision: saved?.dataRevision ?? '0',
+        availability: !covered ? 'missing' : complete ? 'complete' : 'partial',
+        fetchedAt,
+        freshForMs: Number.POSITIVE_INFINITY,
+        work,
+        coverage: {
+          requested: range ? [publicRange(range)] : [],
+          completeRanges: range
+            ? completeCoverage.flatMap((row) => {
+                const from = new Date(
+                  Math.max(row.from.valueOf(), range.from.valueOf()),
+                );
+                const to = new Date(
+                  Math.min(row.to.valueOf(), range.to.valueOf()),
+                );
+                return from < to ? [publicRange({ from, to })] : [];
+              })
+            : [],
+          missingRanges: missing.map(publicRange),
+          asOf: fetchedAt?.toISOString() ?? null,
+          historyBoundary: timestamp(detail?.instrument?.first_1dayCandleDate)
+            ? 'known'
+            : 'unknown',
+          historyState: complete
+            ? 'complete'
+            : work?.state === 'failed'
+              ? 'failed'
+              : work?.state === 'running'
+                ? 'loading'
+                : covered
+                  ? 'partial'
+                  : 'missing',
+        },
+        dependencies: [
+          detail.meta,
+          ...(saved?.coverage ?? []).map((row) =>
+            investmentResourceMeta({
+              resourceKey: `${scope(input)}/${row.from.toISOString()}/${row.to.toISOString()}`,
+              dataRevision: String(row.dataRevision),
+              availability: 'complete',
+              fetchedAt: row.fetchedAt,
+              freshForMs: row.isComplete
+                ? Number.POSITIVE_INFINITY
+                : tailFreshForMs,
+              work: null,
+            }),
+          ),
+        ],
+      }),
+    };
+  }
+
+  prefetch(
+    userId: string,
+    revision: string,
+    positions: { instrumentUid?: string | null; type: string }[],
+  ) {
+    const key = JSON.stringify([
+      userId,
+      revision,
+      positions
+        .map((position) => [position.instrumentUid, position.type])
+        .sort(),
+    ]);
+    const existing = this.prefetches.get(key);
+    if (existing) return existing;
+    const work = this.prefetchPositions(userId, revision, positions).finally(
+      () => this.prefetches.delete(key),
+    );
+    this.prefetches.set(key, work);
+    return work;
+  }
+
+  private async prefetchPositions(
+    userId: string,
+    revision: string,
+    positions: { instrumentUid?: string | null; type: string }[],
+  ) {
+    const now = new Date();
+    const unique = new Map(
+      positions
+        .filter(
+          (position) =>
+            position.instrumentUid && ['STOCK', 'BOND'].includes(position.type),
+        )
+        .map((position) => [position.instrumentUid ?? '', position]),
+    );
+    const details: Awaited<ReturnType<CatalogEnrichmentService['readMany']>> =
+      [];
+    const coverage: Awaited<ReturnType<CandleRepository['coverage']>> = [];
+    for (const instrumentType of ['share', 'bond'] as const) {
+      const uids = [...unique]
+        .filter(
+          ([, position]) =>
+            position.type === (instrumentType === 'share' ? 'STOCK' : 'BOND'),
+        )
+        .map(([uid]) => uid);
+      for (
+        let offset = 0;
+        offset < uids.length;
+        offset += prefetchReadBatchSize
+      ) {
+        const batch = uids.slice(offset, offset + prefetchReadBatchSize);
+        details.push(
+          ...(await this.sources.readMany(userId, instrumentType, batch, true, [
+            'detail',
+          ])),
+        );
+        coverage.push(...(await this.repository.coverage(batch)));
+      }
+    }
+    const byUid = new Map(details.map((detail) => [detail.input.uid, detail]));
+    for (const [uid, position] of unique) {
+      const instrumentType: CandleInput['instrumentType'] =
+        position.type === 'STOCK' ? 'share' : 'bond';
+      const detail = byUid.get(uid);
+      const history = timestamp(detail?.instrument?.first_1dayCandleDate);
+      const plan = planInvestmentScreen(
+        { screen: 'account' },
+        {
+          membershipKnown: true,
+          accounts: [],
+          instruments: [
+            {
+              uid,
+              type: position.type as 'STOCK' | 'BOND',
+              historyFrom: history ? new Date(history) : null,
+            },
+          ],
+        },
+        now,
+      );
+      if (
+        !detail?.instrument ||
+        plan.sources.some((source) => source.kind === 'candles')
+      ) {
+        const input = { uid, instrumentType, period: 'all' as const };
+        const range =
+          detail?.instrument && history
+            ? rangeFor(input, detail.instrument, now)
+            : null;
+        if (
+          !range ||
+          candleGaps(
+            range,
+            coverage.filter((row) => row.instrumentUid === uid),
+            now,
+          ).length
+        )
+          await this.request(userId, revision, input, now, false, 2);
+      }
+    }
+  }
+
+  request(
+    userId: string,
+    revision: string,
+    input: CandleInput,
+    now = new Date(),
+    force = false,
+    priority = 1,
+  ) {
+    const pending = this.reservations.then(async () => {
+      const reserved = await this.repository.reserve(
+        { userId, revision },
+        scope(input),
+        {
+          uid: input.uid,
+          instrumentType: input.instrumentType,
+          ranges: [],
+          period: input.period,
+          now: now.toISOString(),
+          force,
+        },
+        this.runId,
+        this.active,
+        priority,
+      );
+      if (!reserved) return null;
+      const attempt = {
+        userId,
+        revision,
+        id: reserved.work.id,
+        attemptId: reserved.work.attemptId,
+      };
+      if (reserved.launch) {
+        this.active.add(attempt.attemptId);
+        const producer = this.producers.get(scope(input));
+        if (producer) producer.observers.push(attempt);
+        else {
+          const next = { input, attempt, now, force, priority, observers: [] };
+          this.producers.set(scope(input), next);
+          this.queue.push(next);
+        }
+        if (!this.running) {
+          this.running = true;
+          setImmediate(() => {
+            void this.drain();
+          });
+        }
+      }
+      return attempt.id;
+    });
+    this.reservations = pending.catch(() => undefined);
+    return pending;
+  }
+
+  private async load(producer: Producer) {
+    const { input, attempt, now } = producer;
+    const instrument = await this.sources.ensureDetail(
+      attempt.userId,
+      attempt.revision,
+      input.instrumentType,
+      input.uid,
+    );
+    if (!instrument) throw new NotFoundException();
+    const range = rangeFor(input, instrument, now);
+    if (!producer.ranges) {
+      const saved = await this.repository.read(input.uid, range);
+      producer.ranges = (
+        producer.force ? [range] : candleGaps(range, saved.coverage, now)
+      )
+        .flatMap(candleWindows)
+        .reverse();
+      producer.nextWindow = 0;
+    }
+    const ranges = producer.ranges;
+    const index = producer.nextWindow ?? 0;
+    let window = ranges[index];
+    if (window && !producer.force) {
+      const latest = await this.repository.read(input.uid, window);
+      const gaps = candleGaps(window, latest.coverage, now)
+        .flatMap(candleWindows)
+        .reverse();
+      ranges.splice(index, 1, ...gaps);
+      window = ranges[index];
+    }
+    if (window) {
+      const token = await this.access.tokenForAttempt(attempt);
+      if (!token) return false;
+      const candles = await this.tbank.getCandles(
+        token,
+        input.uid,
+        window.from,
+        new Date(Math.min(window.to.valueOf(), now.valueOf())),
+      );
+      const rows = candles.flatMap((candle) => {
+        const time = timestamp(candle.time);
+        if (!time || !candle.close) throw new Error('Invalid candle response');
+        return [
+          {
+            time,
+            close: formatDecimal(decimal(candle.close)),
+            isComplete: candle.isComplete === true,
+          },
+        ];
+      });
+      if (!(await this.repository.stage(attempt, window, rows, now)))
+        return false;
+      if (
+        !producer.force &&
+        !(await this.repository.publish(
+          attempt,
+          { uid: input.uid, ranges: [window] },
+          false,
+        ))
+      )
+        return false;
+      producer.nextWindow = index + 1;
+      if (producer.nextWindow < ranges.length) return null;
+    }
+    return this.repository.publish(attempt, {
+      uid: input.uid,
+      ranges: producer.force ? ranges : [],
+    });
+  }
+
+  private async drain() {
+    try {
+      while (this.queue.length) {
+        this.queue.sort((a, b) => a.priority - b.priority);
+        const producer = this.queue.shift();
+        if (!producer) break;
+        let success = false;
+        let code = 'TBANK_UNAVAILABLE';
+        try {
+          const result = await this.load(producer);
+          if (result === null) {
+            this.queue.push(producer);
+            continue;
+          }
+          success = result;
+        } catch (error) {
+          if (error instanceof Error && error.message === 'INVALID_TBANK_TOKEN')
+            await this.access.rejectMembershipToken(producer.attempt);
+          if (error instanceof ServiceUnavailableException) {
+            const response = error.getResponse();
+            if (
+              typeof response === 'object' &&
+              'code' in response &&
+              typeof response.code === 'string'
+            )
+              code = response.code;
+          }
+        }
+        if (!success) await this.repository.fail(producer.attempt, code);
+        this.active.delete(producer.attempt.attemptId);
+        for (const observer of producer.observers) {
+          if (
+            !success ||
+            !(await this.repository.completeObserver(
+              observer,
+              producer.attempt,
+            ))
+          )
+            await this.repository.fail(observer, code);
+          this.active.delete(observer.attemptId);
+        }
+        this.producers.delete(scope(producer.input));
+        if (success && !producer.force && producer.input.period !== 'all')
+          await this.request(
+            producer.attempt.userId,
+            producer.attempt.revision,
+            { ...producer.input, period: 'all' },
+            producer.now,
+            false,
+            2,
+          );
+      }
+    } catch {
+      this.logger.event('error', 'candles.persistence-failed', {
+        code: 'TBANK_UNAVAILABLE',
+      });
+    } finally {
+      this.active.clear();
+      this.producers.clear();
+      this.queue.length = 0;
+      this.running = false;
+    }
+  }
+}