fix: restore terminal hourly lifecycle after persistence failure (#56)

test/tbank-hourly-lifecycle.e2e-spec.ts

@@ -0,0 +1,525 @@
+import { ConfigService } from '@nestjs/config';
+import assert from 'node:assert/strict';
+import { randomUUID } from 'node:crypto';
+import { setImmediate as immediate } from 'node:timers/promises';
+
+import { TbankAccessService } from '../src/integrations/tbank/tbank-access.service';
+import { TbankTokenService } from '../src/integrations/tbank/tbank-token.service';
+import { CatalogEnrichmentService } from '../src/modules/investments/instruments/application/catalog-enrichment.service';
+import { CatalogDiscoveryRepository } from '../src/modules/investments/instruments/infrastructure/catalog-discovery.repository';
+import { BrokerageOperationsSyncService } from '../src/modules/investments/operations/application/brokerage-operations-sync.service';
+import { BrokerageOperationsRepository } from '../src/modules/investments/operations/infrastructure/brokerage-operations.repository';
+import { TbankHourlyScheduler } from '../src/modules/investments/tbank-sync/tbank-hourly.scheduler';
+import { TbankSyncRepository } from '../src/modules/investments/tbank-sync/tbank-sync.repository';
+import { PrismaService } from '../src/prisma/prisma.service';
+
+type HourlyProgressView = {
+  result?: string;
+  sections: { accountId?: string; errorCode: null | string; section: string }[];
+};
+
+// Real SQLite proves reservation, outcome derivation, recovery and fences;
+// the step owners are controlled so each failure mode can be injected.
+describe('hourly attempt lifecycle (SQLite)', () => {
+  const prisma = new PrismaService(
+    new ConfigService({ DATABASE_URL: process.env.DATABASE_URL }),
+  );
+  const tokens = new TbankTokenService(
+    new ConfigService({
+      TOKEN_ENCRYPTION_KEY: process.env.TOKEN_ENCRYPTION_KEY,
+    }) as never,
+  );
+  const repository = new TbankSyncRepository(prisma);
+  const access = { local: jest.fn() };
+  const membership = { request: jest.fn() };
+  const portfolio = { request: jest.fn() };
+  const operations = { request: jest.fn() };
+  const events = { refresh: jest.fn(), get: jest.fn() };
+  const logger = { event: jest.fn() };
+  const createScheduler = () =>
+    new TbankHourlyScheduler(
+      prisma,
+      repository,
+      access as never,
+      membership as never,
+      portfolio as never,
+      operations as never,
+      events as never,
+      logger as never,
+    );
+  const USER = 'hourly-owner';
+  const REVISION = 'hourly-revision';
+
+  function child(
+    state: 'complete' | 'failed' | 'running',
+    errorCode: null | string,
+    scopeKey: string,
+  ) {
+    return prisma.tbankSyncWork.create({
+      data: {
+        id: randomUUID(),
+        userId: USER,
+        revision: REVISION,
+        scopeKey: `test:${scopeKey}:${randomUUID()}`,
+        attemptId: randomUUID(),
+        processRunId: 'child-process',
+        priority: 0,
+        intent: {},
+        progress: {},
+        state,
+        errorCode,
+      },
+    });
+  }
+
+  function hourlyRow() {
+    return prisma.tbankSyncWork.findFirst({
+      where: { userId: USER, scopeKey: 'hourly' },
+    });
+  }
+
+  async function pump(turns = 20) {
+    for (let i = 0; i < turns; i++) {
+      await immediate();
+      await jest.advanceTimersByTimeAsync(50);
+    }
+  }
+
+  async function runWalk(scheduler: TbankHourlyScheduler, advanceMs = 50) {
+    await scheduler.tick();
+    for (let i = 0; i < 400; i++) {
+      await immediate();
+      await jest.advanceTimersByTimeAsync(advanceMs);
+      const row = await hourlyRow();
+      if (row && row.state !== 'running') return row;
+    }
+    throw new Error('hourly attempt did not settle');
+  }
+
+  beforeAll(async () => {
+    await prisma.$connect();
+    await prisma.user.create({
+      data: { id: USER, email: 'hourly@example.invalid', passwordHash: 'x' },
+    });
+  });
+  afterAll(async () => prisma.$disconnect());
+  beforeEach(async () => {
+    jest.resetAllMocks();
+    await prisma.tbankSyncWork.deleteMany();
+    await prisma.tbankConnection.deleteMany();
+    await prisma.tbankConnection.create({
+      data: {
+        userId: USER,
+        revision: REVISION,
+        encryptedToken: tokens.encrypt('synthetic'),
+        connectedAt: new Date(),
+      },
+    });
+    jest.useFakeTimers({
+      doNotFake: ['queueMicrotask', 'setImmediate', 'nextTick'],
+    });
+    access.local.mockResolvedValue({
+      revision: REVISION,
+      accounts: [{ accountId: 'a' }, { accountId: 'b' }],
+    });
+    membership.request.mockImplementation(async () => {
+      const row = await child('complete', null, 'membership');
+      return { id: row.id, done: Promise.resolve() };
+    });
+    portfolio.request.mockImplementation(
+      async (_userId: string, _revision: string, accountId: string) => {
+        const row = await child('complete', null, `portfolio:${accountId}`);
+        return { id: row.id, done: Promise.resolve() };
+      },
+    );
+    operations.request.mockImplementation(async () => {
+      const row = await child('complete', null, 'operations');
+      return row.id;
+    });
+    events.refresh.mockResolvedValue([]);
+    events.get.mockResolvedValue(undefined);
+  });
+  afterEach(() => {
+    jest.useRealTimers();
+    jest.restoreAllMocks();
+  });
+
+  it('settles the first walk succeeded only when every step row completed', async () => {
+    const row = await runWalk(createScheduler());
+    expect(row.state).toBe('complete');
+    const progress = row.progress as HourlyProgressView;
+    expect(progress.result).toBe('succeeded');
+    expect(progress.sections).toEqual([]);
+    const read = await repository.hourly(USER);
+    expect(read?.state).toBe('succeeded');
+  });
+
+  it('settles partial and keeps successful steps when one account fails', async () => {
+    portfolio.request.mockImplementation(
+      async (_userId: string, _revision: string, accountId: string) => {
+        const failed = accountId === 'b';
+        const row = await child(
+          failed ? 'failed' : 'complete',
+          failed ? 'TBANK_UNAVAILABLE' : null,
+          `portfolio:${accountId}`,
+        );
+        return { id: row.id, done: Promise.resolve() };
+      },
+    );
+    const row = await runWalk(createScheduler());
+    const progress = row.progress as HourlyProgressView;
+    expect(progress.result).toBe('partial');
+    expect(progress.sections).toHaveLength(1);
+    expect(progress.sections[0].section).toBe('portfolio');
+    expect(progress.sections[0].accountId).toBe('b');
+    expect(progress.sections[0].errorCode).toBe('TBANK_UNAVAILABLE');
+  });
+
+  it('settles failed when the membership step fails before any update', async () => {
+    membership.request.mockImplementation(async () => {
+      const row = await child('failed', 'INVALID_TBANK_TOKEN', 'membership');
+      return { id: row.id, done: Promise.resolve() };
+    });
+    const row = await runWalk(createScheduler());
+    const progress = row.progress as HourlyProgressView;
+    expect(progress.result).toBe('failed');
+    expect(row.errorCode).toBe('INVALID_TBANK_TOKEN');
+    expect(progress.sections[0].section).toBe('membership');
+  });
+
+  it('never succeeds when no required step succeeded', async () => {
+    membership.request.mockImplementation(async () => {
+      const row = await child('failed', 'TBANK_UNAVAILABLE', 'membership');
+      return { id: row.id, done: Promise.resolve() };
+    });
+    portfolio.request.mockImplementation(
+      async (_userId: string, _revision: string, accountId: string) => {
+        const row = await child(
+          'failed',
+          'TBANK_UNAVAILABLE',
+          `portfolio:${accountId}`,
+        );
+        return { id: row.id, done: Promise.resolve() };
+      },
+    );
+    operations.request.mockRejectedValue(new Error('db'));
+    events.refresh.mockRejectedValue(new Error('db'));
+    const row = await runWalk(createScheduler());
+    const progress = row.progress as HourlyProgressView;
+    expect(progress.result).toBe('failed');
+    expect(progress.result).not.toBe('succeeded');
+    assert.ok(progress.sections.length > 0);
+  });
+
+  it('caps the outcome at partial when the coordinator itself breaks', async () => {
+    access.local.mockRejectedValueOnce(new Error('db'));
+    const row = await runWalk(createScheduler());
+    const progress = row.progress as HourlyProgressView;
+    expect(progress.result).toBe('partial');
+    expect(row.errorCode).toBe('INTERRUPTED');
+  });
+
+  it('records a section whose dispatch failed as a problem', async () => {
+    events.refresh.mockRejectedValue(new Error('db'));
+    const row = await runWalk(createScheduler());
+    const progress = row.progress as HourlyProgressView;
+    expect(progress.result).toBe('partial');
+    const eventsSection = progress.sections.find(
+      (entry) => entry.section === 'events',
+    );
+    assert.ok(eventsSection);
+    expect(eventsSection.errorCode).toBe('TBANK_UNAVAILABLE');
+    expect(row.errorCode).toBe('TBANK_UNAVAILABLE');
+  });
+
+  it('closes a stuck step as DEADLINE and keeps its row running', async () => {
+    operations.request.mockImplementation(async () => {
+      const row = await child('running', null, 'operations');
+      return row.id;
+    });
+    const scheduler = createScheduler();
+    await scheduler.tick();
+    await pump();
+    await jest.advanceTimersByTimeAsync(16 * 60 * 1000);
+    await pump(20);
+    const row = await hourlyRow();
+    assert.ok(row);
+    expect(row.state).not.toBe('running');
+    const progress = row.progress as HourlyProgressView;
+    expect(progress.result).toBe('partial');
+    assert.ok(
+      progress.sections.some(
+        (entry) =>
+          entry.section === 'operations' && entry.errorCode === 'DEADLINE',
+      ),
+    );
+    const stuck = await prisma.tbankSyncWork.findFirst({
+      where: { userId: USER, state: 'running' },
+    });
+    assert.ok(stuck, 'the wait bound does not cancel the upstream call');
+  });
+
+  it('recovers a lost settle and unblocks the next attempt at its term', async () => {
+    const settleSpy = jest
+      .spyOn(repository, 'settleHourly')
+      .mockRejectedValueOnce(new Error('db'));
+    const scheduler = createScheduler();
+    await scheduler.tick();
+    for (let i = 0; i < 40 && settleSpy.mock.calls.length === 0; i++) {
+      await immediate();
+      await jest.advanceTimersByTimeAsync(50);
+    }
+    expect(settleSpy.mock.calls).toHaveLength(1);
+    const open = await hourlyRow();
+    expect(open?.state).toBe('running');
+    await pump(20);
+    await scheduler.tick();
+    await pump(20);
+    const closed = await hourlyRow();
+    expect(closed?.state).toBe('complete');
+    expect((closed?.progress as HourlyProgressView).result).toBe('partial');
+    expect(closed?.errorCode).toBe('INTERRUPTED');
+    await jest.advanceTimersByTimeAsync(61 * 60 * 1000);
+    await scheduler.tick();
+    await pump(30);
+    const next = await hourlyRow();
+    expect(next?.attemptId).not.toBe(closed?.attemptId);
+    expect(next?.state).not.toBe('running');
+    expect((next?.progress as HourlyProgressView).result).toBe('succeeded');
+  });
+
+  it('closes an abandoned foreign-process attempt and keeps the normal term', async () => {
+    const stuck = await child('complete', null, 'membership');
+    const deadAttemptId = randomUUID();
+    await prisma.tbankSyncWork.create({
+      data: {
+        id: randomUUID(),
+        userId: USER,
+        revision: REVISION,
+        scopeKey: 'hourly',
+        attemptId: deadAttemptId,
+        processRunId: 'dead-process',
+        priority: 0,
+        intent: {},
+        state: 'running',
+        progress: {
+          children: [{ id: stuck.id, section: 'membership' }],
+          attempts: { [stuck.id]: stuck.attemptId },
+          sections: [],
+        },
+      },
+    });
+    const scheduler = createScheduler();
+    await runWalk(scheduler);
+    const closed = await hourlyRow();
+    expect(closed?.attemptId).toBe(deadAttemptId);
+    expect(closed?.state).not.toBe('running');
+    expect((closed?.progress as HourlyProgressView).result).toBe('partial');
+    expect(closed?.errorCode).toBe('INTERRUPTED');
+    const hourlyCount = await prisma.tbankSyncWork.count({
+      where: { userId: USER, scopeKey: 'hourly' },
+    });
+    expect(hourlyCount).toBe(1);
+    await jest.advanceTimersByTimeAsync(61 * 60 * 1000);
+    await scheduler.tick();
+    await pump(30);
+    const next = await hourlyRow();
+    expect(next?.attemptId).not.toBe(deadAttemptId);
+    expect(next?.state).not.toBe('running');
+  });
+
+  it('does not overlap a running attempt with a second walk', async () => {
+    operations.request.mockImplementation(async () => {
+      const row = await child('running', null, 'operations');
+      return row.id;
+    });
+    const scheduler = createScheduler();
+    await scheduler.tick();
+    await pump();
+    expect((await hourlyRow())?.state).toBe('running');
+    await scheduler.tick();
+    await pump();
+    const hourlyCount = await prisma.tbankSyncWork.count({
+      where: { userId: USER, scopeKey: 'hourly' },
+    });
+    expect(hourlyCount).toBe(1);
+    expect((await hourlyRow())?.state).toBe('running');
+    await jest.advanceTimersByTimeAsync(16 * 60 * 1000);
+    await pump(20);
+    expect((await hourlyRow())?.state).not.toBe('running');
+  });
+
+  it('does not start a new attempt before the next hourly term', async () => {
+    const scheduler = createScheduler();
+    const row = await runWalk(scheduler);
+    const firstAttemptId = row.attemptId;
+    expect(row.state).toBe('complete');
+    await scheduler.tick();
+    expect((await hourlyRow())?.attemptId).toBe(firstAttemptId);
+    await jest.advanceTimersByTimeAsync(59 * 60 * 1000);
+    await scheduler.tick();
+    expect((await hourlyRow())?.attemptId).toBe(firstAttemptId);
+    await jest.advanceTimersByTimeAsync(2 * 60 * 1000);
+    await scheduler.tick();
+    await pump(30);
+    expect((await hourlyRow())?.attemptId).not.toBe(firstAttemptId);
+  });
+
+  it('counts a late step result before the attempt outcome is recorded', async () => {
+    const late: { id: string }[] = [];
+    operations.request.mockImplementation(async () => {
+      const row = await child('running', null, 'operations');
+      late.push(row);
+      return row.id;
+    });
+    const scheduler = createScheduler();
+    await scheduler.tick();
+    await pump();
+    assert.ok(late.length > 0);
+    for (const row of late) {
+      await prisma.tbankSyncWork.update({
+        where: { id: row.id },
+        data: { state: 'complete' },
+      });
+    }
+    let settled = false;
+    for (let i = 0; i < 100 && !settled; i++) {
+      await immediate();
+      await jest.advanceTimersByTimeAsync(200);
+      const current = await hourlyRow();
+      settled = Boolean(current && current.state !== 'running');
+    }
+    const row = await hourlyRow();
+    expect(row).not.toBeNull();
+    expect(row?.state).not.toBe('running');
+    expect((row?.progress as HourlyProgressView).result).toBe('succeeded');
+  });
+
+  it('treats a step superseded by a successor as stale and keeps CAS fencing', async () => {
+    const stale = await child('complete', null, 'membership');
+    const attemptId = randomUUID();
+    const id = randomUUID();
+    await prisma.tbankSyncWork.create({
+      data: {
+        id,
+        userId: USER,
+        revision: REVISION,
+        scopeKey: 'hourly',
+        attemptId,
+        processRunId: 'dead-process',
+        priority: 0,
+        intent: {},
+        state: 'running',
+        progress: {
+          children: [{ id: stale.id, section: 'membership' }],
+          attempts: { [stale.id]: randomUUID() },
+          sections: [],
+        },
+      },
+    });
+    const foreign = await repository.settleHourly(
+      { userId: USER, revision: REVISION, id, attemptId: randomUUID() },
+      'INTERRUPTED',
+    );
+    expect(foreign.count).toBe(0);
+    expect((await hourlyRow())?.state).toBe('running');
+    const closed = await repository.settleHourly(
+      { userId: USER, revision: REVISION, id, attemptId },
+      'INTERRUPTED',
+    );
+    expect(closed.count).toBe(1);
+    const progress = (await hourlyRow())?.progress as HourlyProgressView;
+    expect(progress.result).toBe('failed');
+    expect(progress.sections[0].errorCode).toBeNull();
+    const late = await repository.settleHourly(
+      { userId: USER, revision: REVISION, id, attemptId },
+      'INTERRUPTED',
+    );
+    expect(late.count).toBe(0);
+  });
+
+  it('creates no reservations after a controlled shutdown', async () => {
+    const scheduler = createScheduler();
+    scheduler.onModuleInit();
+    scheduler.onModuleDestroy();
+    await jest.advanceTimersByTimeAsync(2 * 60 * 60 * 1000);
+    await pump(5);
+    expect(await prisma.tbankSyncWork.count({ where: { userId: USER } })).toBe(0);
+  });
+
+  it('closes the operations step durably when its finalizer cannot record', async () => {
+    await prisma.tbankConnection.update({
+      where: { userId: USER },
+      data: { membershipAt: new Date() },
+    });
+    await prisma.tbankAccount.create({
+      data: {
+        userId: USER,
+        revision: REVISION,
+        accountId: 'a',
+        type: 'BROKER',
+        name: 'Test',
+        upstreamOrder: 0,
+      },
+    });
+    const accessReal = new TbankAccessService(prisma, tokens);
+    const operationsRepository = new BrokerageOperationsRepository(prisma);
+    const sync = new BrokerageOperationsSyncService(
+      operationsRepository,
+      accessReal,
+      { getOperationsByCursor: jest.fn() } as never,
+      logger as never,
+    );
+    jest
+      .spyOn(operationsRepository, 'pending')
+      .mockRejectedValueOnce(new Error('db'));
+    jest
+      .spyOn(operationsRepository, 'fail')
+      .mockRejectedValueOnce(new Error('db'));
+    const id = await sync.request(
+      USER,
+      [
+        {
+          accountId: 'a',
+          from: new Date(Date.UTC(2026, 0, 1)),
+          to: new Date(),
+        },
+      ] as never,
+      true,
+      'screen',
+      REVISION,
+    );
+    assert.ok(id);
+    await pump(40);
+    const row = await prisma.tbankSyncWork.findFirst({
+      where: { userId: USER, id },
+    });
+    expect(row?.state).toBe('failed');
+  });
+
+  it('finishes open catalog steps durably when publication fails mid-batch', async () => {
+    const accessReal = new TbankAccessService(prisma, tokens);
+    const catalogRepository = new CatalogDiscoveryRepository(prisma);
+    const enrichment = new CatalogEnrichmentService(
+      catalogRepository,
+      accessReal,
+      { getLastPrices: jest.fn().mockResolvedValue([]) } as never,
+      logger as never,
+      {} as never,
+      {} as never,
+      {} as never,
+    );
+    jest
+      .spyOn(catalogRepository, 'publishSource')
+      .mockRejectedValue(new Error('db'));
+    await enrichment.requestMany(USER, REVISION, 'share', ['uid'], false, [
+      'last-price',
+    ]);
+    await pump(40);
+    const rows = await prisma.tbankSyncWork.findMany({
+      where: { userId: USER, scopeKey: { not: 'hourly' } },
+    });
+    assert.ok(rows.length > 0);
+    for (const row of rows) expect(row.state).toBe('failed');
+  });
+});