Enhance distributed transaction handling and order fulfillment orchestration

- Introduced retry logic in DistributedTransactionService to improve resilience during step execution, allowing for configurable maximum retries.
- Refactored order fulfillment orchestration to include step status updates, enhancing visibility into the execution process and error handling.
- Implemented createTrackedStep utility to streamline step execution and status management, ensuring consistent handling of success and failure states.
- Improved toast message handling in usePaymentRefresh and SimFeatureToggles components for better user feedback during asynchronous operations.
- Added cleanup logic in SimManagementSection to prevent memory leaks and ensure proper aborting of ongoing requests.

apps/bff/src/core/database/services/distributed-transaction.service.ts

@@ -103,9 +103,10 @@ export class DistributedTransactionService {
     const {
       description,
       timeout = 120000, // 2 minutes default for distributed operations
-      maxRetries: _maxRetries = 1, // Less retries for distributed operations
+      maxRetries: configuredMaxRetries = 1,
       continueOnNonCriticalFailure = false,
     } = options;
+    const maxRetries = Math.max(0, Math.floor(configuredMaxRetries ?? 0));
 
     const transactionId = this.generateTransactionId();
     const startTime = Date.now();
@@ -130,7 +131,7 @@ export class DistributedTransactionService {
 
         try {
           const stepStartTime = Date.now();
-          const result = await this.executeStepWithTimeout(step, timeout);
+          const result = await this.executeStepWithRetry(step, timeout, maxRetries, transactionId);
           const stepDuration = Date.now() - stepStartTime;
 
           const key = step.id as keyof StepResultMap<TSteps>;
@@ -402,6 +403,36 @@ export class DistributedTransactionService {
     return rollbacksExecuted;
   }
 
+  private async executeStepWithRetry<TResult>(
+    step: DistributedStep<string, TResult>,
+    timeout: number,
+    maxRetries: number,
+    transactionId: string
+  ): Promise<TResult> {
+    const totalAttempts = step.retryable ? Math.max(1, maxRetries + 1) : 1;
+    let attempt = 0;
+
+    // attempt counter represents completed attempts; loop until success or attempts exhausted
+    while (attempt < totalAttempts) {
+      try {
+        return await this.executeStepWithTimeout(step, timeout);
+      } catch (error) {
+        attempt++;
+        if (attempt >= totalAttempts) {
+          throw error;
+        }
+
+        this.logger.warn(`Retrying step: ${step.id} [${transactionId}]`, {
+          attempt: attempt + 1,
+          maxAttempts: totalAttempts,
+          error: getErrorMessage(error),
+        });
+      }
+    }
+
+    throw new Error(`Step ${step.id} failed after ${totalAttempts} attempts`);
+  }
+
   private generateTransactionId(): string {
     return `dtx_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`;
   }

apps/bff/src/modules/orders/services/order-fulfillment-orchestrator.service.ts

@@ -98,11 +98,13 @@ export class OrderFulfillmentOrchestrator {
 
     try {
       // Step 1: Validation (no rollback needed)
+      this.updateStepStatus(context, "validation", "in_progress");
       try {
         context.validation = await this.orderFulfillmentValidator.validateFulfillmentRequest(
           sfOrderId,
           idempotencyKey
         );
+        this.updateStepStatus(context, "validation", "completed");
 
         if (context.validation.isAlreadyProvisioned) {
           this.logger.log("Order already provisioned, skipping fulfillment", { sfOrderId });
@@ -122,6 +124,7 @@ export class OrderFulfillmentOrchestrator {
           return context;
         }
       } catch (error) {
+        this.updateStepStatus(context, "validation", "failed", getErrorMessage(error));
         this.logger.error("Fulfillment validation failed", {
           sfOrderId,
           error: getErrorMessage(error),
@@ -157,7 +160,7 @@ export class OrderFulfillmentOrchestrator {
           {
             id: "sf_status_update",
             description: "Update Salesforce order status to Activating",
-            execute: async () => {
+            execute: this.createTrackedStep(context, "sf_status_update", async () => {
               const result = await this.salesforceService.updateOrder({
                 Id: sfOrderId,
                 Activation_Status__c: "Activating",
@@ -171,7 +174,7 @@ export class OrderFulfillmentOrchestrator {
                 timestamp: new Date().toISOString(),
               });
               return result;
-            },
+            }),
             rollback: async () => {
               await this.salesforceService.updateOrder({
                 Id: sfOrderId,
@@ -183,13 +186,15 @@ export class OrderFulfillmentOrchestrator {
           {
             id: "order_details",
             description: "Retain order details in context",
-            execute: () => Promise.resolve(context.orderDetails),
+            execute: this.createTrackedStep(context, "order_details", () =>
+              Promise.resolve(context.orderDetails)
+            ),
             critical: false,
           },
           {
             id: "mapping",
             description: "Map OrderItems to WHMCS format",
-            execute: () => {
+            execute: this.createTrackedStep(context, "mapping", () => {
               if (!context.orderDetails) {
                 return Promise.reject(new Error("Order details are required for mapping"));
               }
@@ -204,13 +209,13 @@ export class OrderFulfillmentOrchestrator {
               });
 
               return Promise.resolve(result);
-            },
+            }),
             critical: true,
           },
           {
             id: "whmcs_create",
             description: "Create order in WHMCS",
-            execute: async () => {
+            execute: this.createTrackedStep(context, "whmcs_create", async () => {
               if (!context.validation) {
                 throw new OrderValidationException("Validation context is missing", {
                   sfOrderId,
@@ -242,7 +247,7 @@ export class OrderFulfillmentOrchestrator {
 
               whmcsCreateResult = result;
               return result;
-            },
+            }),
             rollback: () => {
               if (whmcsCreateResult?.orderId) {
                 // Note: WHMCS doesn't have an automated cancel API
@@ -263,7 +268,7 @@ export class OrderFulfillmentOrchestrator {
           {
             id: "whmcs_accept",
             description: "Accept/provision order in WHMCS",
-            execute: async () => {
+            execute: this.createTrackedStep(context, "whmcs_accept", async () => {
               if (!whmcsCreateResult?.orderId) {
                 throw new WhmcsOperationException("WHMCS order ID missing before acceptance step", {
                   sfOrderId,
@@ -273,7 +278,7 @@ export class OrderFulfillmentOrchestrator {
 
               await this.whmcsOrderService.acceptOrder(whmcsCreateResult.orderId, sfOrderId);
               return { orderId: whmcsCreateResult.orderId };
-            },
+            }),
             rollback: () => {
               if (whmcsCreateResult?.orderId) {
                 // Note: WHMCS doesn't have an automated cancel API for accepted orders
@@ -295,7 +300,7 @@ export class OrderFulfillmentOrchestrator {
           {
             id: "sim_fulfillment",
             description: "SIM-specific fulfillment (if applicable)",
-            execute: async () => {
+            execute: this.createTrackedStep(context, "sim_fulfillment", async () => {
               if (context.orderDetails?.orderType === "SIM") {
                 const configurations = this.extractConfigurations(payload.configurations);
                 await this.simFulfillmentService.fulfillSimOrder({
@@ -305,13 +310,13 @@ export class OrderFulfillmentOrchestrator {
                 return { completed: true as const };
               }
               return { skipped: true as const };
-            },
+            }),
             critical: false, // SIM fulfillment failure shouldn't rollback the entire order
           },
           {
             id: "sf_success_update",
             description: "Update Salesforce with success",
-            execute: async () => {
+            execute: this.createTrackedStep(context, "sf_success_update", async () => {
               const result = await this.salesforceService.updateOrder({
                 Id: sfOrderId,
                 Status: "Completed",
@@ -331,7 +336,7 @@ export class OrderFulfillmentOrchestrator {
                 },
               });
               return result;
-            },
+            }),
             rollback: async () => {
               await this.salesforceService.updateOrder({
                 Id: sfOrderId,
@@ -529,4 +534,44 @@ export class OrderFulfillmentOrchestrator {
       steps: context.steps,
     };
   }
+
+  private updateStepStatus(
+    context: OrderFulfillmentContext,
+    stepName: string,
+    status: OrderFulfillmentStep["status"],
+    error?: string
+  ): void {
+    const step = context.steps.find(s => s.step === stepName);
+    if (!step) return;
+
+    const timestamp = new Date();
+    if (status === "in_progress") {
+      step.status = "in_progress";
+      step.startedAt = timestamp;
+      step.error = undefined;
+      return;
+    }
+
+    step.status = status;
+    step.completedAt = timestamp;
+    step.error = status === "failed" ? error : undefined;
+  }
+
+  private createTrackedStep<TResult>(
+    context: OrderFulfillmentContext,
+    stepName: string,
+    executor: () => Promise<TResult>
+  ): () => Promise<TResult> {
+    return async () => {
+      this.updateStepStatus(context, stepName, "in_progress");
+      try {
+        const result = await executor();
+        this.updateStepStatus(context, stepName, "completed");
+        return result;
+      } catch (error) {
+        this.updateStepStatus(context, stepName, "failed", getErrorMessage(error));
+        throw error;
+      }
+    };
+  }
 }

apps/portal/src/features/billing/hooks/usePaymentRefresh.ts

@@ -1,6 +1,6 @@
 "use client";
 
-import { useCallback, useEffect, useState } from "react";
+import { useCallback, useEffect, useRef, useState } from "react";
 import { apiClient } from "@/lib/api";
 import { paymentMethodListSchema, type PaymentMethodList } from "@customer-portal/domain/payments";
 import { useAuthSession } from "@/features/auth/services/auth.store";
@@ -22,12 +22,20 @@ export function usePaymentRefresh({
   hasMethods,
 }: UsePaymentRefreshOptions) {
   const { isAuthenticated } = useAuthSession();
+  const hideToastTimeout = useRef<ReturnType<typeof setTimeout> | null>(null);
   const [toast, setToast] = useState<{ visible: boolean; text: string; tone: Tone }>({
     visible: false,
     text: "",
     tone: "info",
   });
 
+  const clearHideToastTimeout = () => {
+    if (hideToastTimeout.current) {
+      clearTimeout(hideToastTimeout.current);
+      hideToastTimeout.current = null;
+    }
+  };
+
   const triggerRefresh = useCallback(async () => {
     // Don't trigger refresh if not authenticated
     if (!isAuthenticated) {
@@ -57,10 +65,20 @@ export function usePaymentRefresh({
     } catch {
       setToast({ visible: true, text: "Could not refresh payment methods", tone: "warning" });
     } finally {
-      setTimeout(() => setToast(t => ({ ...t, visible: false })), 2200);
+      clearHideToastTimeout();
+      hideToastTimeout.current = window.setTimeout(() => {
+        setToast(t => ({ ...t, visible: false }));
+        hideToastTimeout.current = null;
+      }, 2200);
     }
   }, [isAuthenticated, refetch, hasMethods]);
 
+  useEffect(() => {
+    return () => {
+      clearHideToastTimeout();
+    };
+  }, []);
+
   useEffect(() => {
     if (!attachFocusListeners) return;
 

apps/portal/src/features/sim-management/components/SimFeatureToggles.tsx

@@ -1,6 +1,6 @@
 "use client";
 
-import React, { useEffect, useMemo, useState } from "react";
+import React, { useEffect, useMemo, useRef, useState } from "react";
 import { apiClient } from "@/lib/api";
 
 interface SimFeatureTogglesProps {
@@ -41,6 +41,7 @@ export function SimFeatureToggles({
   const [loading, setLoading] = useState(false);
   const [error, setError] = useState<string | null>(null);
   const [success, setSuccess] = useState<string | null>(null);
+  const successTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
 
   useEffect(() => {
     setVm(initial.vm);
@@ -50,6 +51,10 @@ export function SimFeatureToggles({
   }, [initial.vm, initial.cw, initial.ir, initial.nt]);
 
   const reset = () => {
+    if (successTimerRef.current) {
+      clearTimeout(successTimerRef.current);
+      successTimerRef.current = null;
+    }
     setVm(initial.vm);
     setCw(initial.cw);
     setIr(initial.ir);
@@ -87,10 +92,25 @@ export function SimFeatureToggles({
       setError(e instanceof Error ? e.message : "Failed to submit changes");
     } finally {
       setLoading(false);
-      setTimeout(() => setSuccess(null), 3000);
+      if (successTimerRef.current) {
+        clearTimeout(successTimerRef.current);
+      }
+      successTimerRef.current = window.setTimeout(() => {
+        setSuccess(null);
+        successTimerRef.current = null;
+      }, 3000);
     }
   };
 
+  useEffect(() => {
+    return () => {
+      if (successTimerRef.current) {
+        clearTimeout(successTimerRef.current);
+        successTimerRef.current = null;
+      }
+    };
+  }, []);
+
   return (
     <div className="space-y-6">
       {/* Service Options */}

apps/portal/src/features/sim-management/components/SimManagementSection.tsx

@@ -1,6 +1,6 @@
 "use client";
 
-import React, { useState, useEffect, useCallback } from "react";
+import React, { useState, useEffect, useCallback, useRef } from "react";
 import {
   DevicePhoneMobileIcon,
   ExclamationTriangleIcon,
@@ -21,13 +21,30 @@ export function SimManagementSection({ subscriptionId }: SimManagementSectionPro
   const [simInfo, setSimInfo] = useState<SimInfo | null>(null);
   const [loading, setLoading] = useState(true);
   const [error, setError] = useState<string | null>(null);
+  const abortControllerRef = useRef<AbortController | null>(null);
+  const isMountedRef = useRef(true);
+
+  useEffect(() => {
+    return () => {
+      isMountedRef.current = false;
+      abortControllerRef.current?.abort();
+    };
+  }, []);
 
   const fetchSimInfo = useCallback(async () => {
-    try {
+    abortControllerRef.current?.abort();
-      setError(null);
+    const controller = new AbortController();
+    abortControllerRef.current = controller;
 
+    if (isMountedRef.current) {
+      setLoading(true);
+      setError(null);
+    }
+
+    try {
       const response = await apiClient.GET<SimInfo>("/api/subscriptions/{id}/sim", {
         params: { path: { id: subscriptionId } },
+        signal: controller.signal,
       });
 
       if (!response.data) {
@@ -36,8 +53,15 @@ export function SimManagementSection({ subscriptionId }: SimManagementSectionPro
 
       const payload = simInfoSchema.parse(response.data);
 
+      if (controller.signal.aborted || !isMountedRef.current) {
+        return;
+      }
+
       setSimInfo(payload);
     } catch (err: unknown) {
+      if (controller.signal.aborted || !isMountedRef.current) {
+        return;
+      }
       const hasStatus = (value: unknown): value is { status: number } =>
         typeof value === "object" &&
         value !== null &&
@@ -52,6 +76,9 @@ export function SimManagementSection({ subscriptionId }: SimManagementSectionPro
 
       setError(err instanceof Error ? err.message : "Failed to load SIM information");
     } finally {
+      if (controller.signal.aborted || !isMountedRef.current) {
+        return;
+      }
       setLoading(false);
     }
   }, [subscriptionId]);
@@ -61,7 +88,6 @@ export function SimManagementSection({ subscriptionId }: SimManagementSectionPro
   }, [fetchSimInfo]);
 
   const handleRefresh = () => {
-    setLoading(true);
     void fetchSimInfo();
   };