How to Monitor Sage with Vigilmon

Sage is an Elixir library that implements the saga pattern for distributed multi-step transactions. Each step in a Sage saga declares both a forward action and a compensating transaction. When any step fails, Sage automatically runs the compensation functions in reverse order — rolling back charges, releasing reservations, and reverting state changes. When a compensation itself fails — because the payment gateway is unreachable, an external API returns an unexpected error, or a database is temporarily unavailable — Sage logs the failure and continues the compensation chain, but your transaction is now in a partially compensated state. Vigilmon adds external uptime and heartbeat monitoring so you know when your sagas stop completing cleanly, not just when you accidentally discover a customer's account in an inconsistent state.

What You'll Set Up

• HTTP health endpoint surfacing saga success, failure, and compensation error rates
• Heartbeat monitor for scheduled saga-driven operations
• :telemetry handler capturing saga outcomes
• Slack alerts when saga compensation failures occur

Prerequisites

• Elixir 1.14+ with sage in mix.exs
• A Phoenix or OTP application running Sage sagas
• A free Vigilmon account

Step 1: Track Saga Outcomes with a Telemetry Handler

Sage emits telemetry events at saga completion. Hook into them to track outcomes:

# lib/my_app/saga_tracker.ex
defmodule MyApp.SagaTracker do
  use Agent

  def start_link(_opts) do
    Agent.start_link(fn -> %{} end, name: __MODULE__)
  end

  def record(saga_name, outcome) when outcome in [:success, :failure, :compensation_error] do
    Agent.update(__MODULE__, fn state ->
      Map.update(state, saga_name, new_counts(outcome), fn counts ->
        Map.update(counts, outcome, 1, &(&1 + 1))
      end)
    end)
  end

  defp new_counts(outcome), do: %{success: 0, failure: 0, compensation_error: 0} |> Map.put(outcome, 1)

  def stats, do: Agent.get(__MODULE__, & &1)
end

Add to your supervision tree:

# lib/my_app/application.ex
children = [
  MyApp.Repo,
  MyAppWeb.Endpoint,
  MyApp.SagaTracker,
  # ...
]

Attach a telemetry handler for Sage events:

# lib/my_app/application.ex
def start(_type, _args) do
  :telemetry.attach_many(
    "sage-saga-tracker",
    [
      [:sage, :transaction, :stop],
      [:sage, :compensation, :exception],
    ],
    &MyApp.SagaTelemetryHandler.handle_event/4,
    nil
  )

  # ... supervisor start
end

# lib/my_app/saga_telemetry_handler.ex
defmodule MyApp.SagaTelemetryHandler do
  require Logger

  def handle_event([:sage, :transaction, :stop], _measurements, metadata, _config) do
    saga_name = metadata[:name] || :unknown

    case metadata[:result] do
      {:ok, _} ->
        MyApp.SagaTracker.record(saga_name, :success)

      {:error, _failed_step, _effects, _compensations} ->
        # All compensations ran (even if some had errors)
        # Check if compensation errors occurred
        compensation_errors = metadata[:compensation_errors] || []
        outcome = if compensation_errors == [], do: :failure, else: :compensation_error
        MyApp.SagaTracker.record(saga_name, outcome)

        if outcome == :compensation_error do
          Logger.error(
            "Saga #{inspect(saga_name)} has unresolved compensation errors: " <>
              inspect(compensation_errors)
          )
        end
    end
  end

  def handle_event([:sage, :compensation, :exception], _measurements, metadata, _config) do
    saga_name = metadata[:name] || :unknown
    Logger.error("Saga #{inspect(saga_name)} compensation raised: #{inspect(metadata[:error])}")
    MyApp.SagaTracker.record(saga_name, :compensation_error)
  end
end

Step 2: Define a Sage Saga with Compensating Transactions

# lib/my_app/sagas/checkout_saga.ex
defmodule MyApp.Sagas.CheckoutSaga do
  alias MyApp.{Orders, Inventory, Payments, Notifications}

  def run(attrs) do
    Sage.new()
    |> Sage.run(:reserve_inventory, &reserve_inventory/2, &release_inventory/3)
    |> Sage.run(:create_order, &create_order/2, &cancel_order/3)
    |> Sage.run(:charge_payment, &charge_payment/2, &refund_payment/3)
    |> Sage.run(:send_confirmation, &send_confirmation/2, &noop_compensation/3)
    |> Sage.transaction(MyApp.Repo, attrs)
  end

  defp reserve_inventory(%{items: items}, _effects) do
    case Inventory.reserve(items) do
      {:ok, reservation_id} -> {:ok, reservation_id}
      {:error, :insufficient_stock} -> {:error, :out_of_stock}
    end
  end

  defp release_inventory(_attrs, _effects, {:reserve_inventory, reservation_id}) do
    Inventory.release(reservation_id)
  end

  defp create_order(attrs, %{reserve_inventory: reservation_id}) do
    Orders.create(Map.put(attrs, :reservation_id, reservation_id))
  end

  defp cancel_order(_attrs, _effects, {:create_order, order}) do
    Orders.cancel(order.id)
  end

  defp charge_payment(%{user_id: uid, total: total}, %{create_order: order}) do
    case Payments.charge(uid, total, order.id) do
      {:ok, charge} -> {:ok, charge}
      {:error, reason} -> {:error, {:payment_failed, reason}}
    end
  end

  defp refund_payment(_attrs, _effects, {:charge_payment, charge}) do
    Payments.refund(charge.id)
  end

  defp send_confirmation(%{user_id: uid}, %{create_order: order}) do
    Notifications.send_order_confirmation(uid, order)
    {:ok, :sent}
  end

  defp noop_compensation(_attrs, _effects, _failure), do: :ok
end

Step 3: Add a Health Endpoint

# lib/my_app_web/controllers/health_controller.ex
defmodule MyAppWeb.HealthController do
  use MyAppWeb, :controller

  alias MyApp.SagaTracker

  def index(conn, _params) do
    stats = SagaTracker.stats()

    # Flag unhealthy if any saga has compensation errors
    has_compensation_errors =
      stats
      |> Map.values()
      |> Enum.any?(fn counts -> Map.get(counts, :compensation_error, 0) > 0 end)

    checks = %{
      sagas: stats,
      compensation_errors_present: has_compensation_errors,
      database: check_database()
    }

    status = if !has_compensation_errors and checks.database == :ok, do: 200, else: 503

    conn
    |> put_status(status)
    |> json(%{
      status: if(status == 200, do: "ok", else: "degraded"),
      checks: checks
    })
  end

  defp check_database do
    case Ecto.Adapters.SQL.query(MyApp.Repo, "SELECT 1", []) do
      {:ok, _} -> :ok
      _ -> :error
    end
  end
end

Register the route:

# lib/my_app_web/router.ex
scope "/", MyAppWeb do
  get "/health", HealthController, :index
end

Test locally:

curl -s http://localhost:4000/health | jq .
# {
#   "status": "ok",
#   "checks": {
#     "compensation_errors_present": false,
#     "database": "ok"
#   }
# }

Step 4: Set Up HTTP Monitoring in Vigilmon

1. Sign in at vigilmon.online
2. Click New Monitor → HTTP
3. URL: https://yourdomain.com/health
4. Expected status: 200
5. Add a keyword check: "compensation_errors_present":false — this fires when Sage's rollback mechanisms themselves start failing
6. Interval: 1 minute
7. Save

Compensation errors are particularly dangerous because they indicate your system is in a partially inconsistent state — Vigilmon catching this immediately lets you investigate before customers are impacted.

Step 5: Heartbeat Monitoring for Saga-Driven Batch Operations

For nightly reconciliation sagas or scheduled cleanup operations, add a heartbeat:

# lib/my_app/workers/reconciliation_worker.ex
defmodule MyApp.Workers.ReconciliationWorker do
  use Oban.Worker, queue: :reconciliation, max_attempts: 1

  require Logger

  @impl Oban.Worker
  def perform(%Oban.Job{}) do
    case MyApp.Sagas.ReconciliationSaga.run(%{date: Date.utc_today()}) do
      {:ok, _result} ->
        ping_heartbeat()
        :ok

      {:error, failed_step, _effects, compensation_errors} ->
        Logger.error(
          "Reconciliation saga failed at #{inspect(failed_step)}, " <>
            "#{length(compensation_errors)} compensation errors"
        )
        {:error, :reconciliation_failed}
    end
  end

  defp ping_heartbeat do
    url = Application.get_env(:my_app, :vigilmon)[:reconciliation_heartbeat_url]

    if url do
      case Req.get(url, receive_timeout: 5_000) do
        {:ok, _} -> :ok
        {:error, reason} ->
          Logger.warning("Vigilmon heartbeat ping failed: #{inspect(reason)}")
      end
    end
  end
end

Configure:

# config/runtime.exs
config :my_app, :vigilmon,
  reconciliation_heartbeat_url: System.get_env("VIGILMON_RECONCILIATION_HEARTBEAT_URL")

In Vigilmon:

1. Click New Monitor → Heartbeat
2. Name: reconciliation-saga
3. Expected interval: 24 hours
4. Copy the ping URL and set it as VIGILMON_RECONCILIATION_HEARTBEAT_URL

Step 6: Alert on Compensation Failures via Slack

1. In Vigilmon: Notifications → New Channel → Slack
2. Paste your Slack incoming webhook URL
3. Enable it on your /health monitor and the reconciliation heartbeat

A compensation error alert:

🔴 DOWN: yourdomain.com/health
Keyword "compensation_errors_present":false not found
Detected: EU-West, US-East
Possible cause: saga rollback failure — check for orphaned charges or reservations

What You've Built

| What | How | |------|-----| | Saga outcome tracking | Telemetry handler capturing Sage stop and compensation:exception events | | Compensation error detection | Health check with keyword monitor on compensation_errors_present | | Scheduled saga monitoring | Heartbeat on nightly reconciliation Oban worker | | Compensation failure alerts | Slack notification when partial rollback occurs | | Database health | SQL ping in health check ensuring Ecto is available for saga persistence |

Sage ensures your multi-step transactions roll back cleanly. Vigilmon ensures you know immediately when the rollback itself needs attention.

Next Steps

• Add a Vigilmon heartbeat per critical saga (checkout, refund, account creation) for granular visibility
• Extend the health endpoint to expose per-saga failure rates, not just compensation error presence
• Set up a separate Slack channel for compensation errors — these warrant higher urgency than normal failures
• Use Vigilmon's status page feature to communicate saga-related service degradation to customers

Get started free at vigilmon.online.