How to Monitor ExHashRing with Vigilmon

ExHashRing is a consistent hashing library for Elixir that implements a virtual node ring. It maps keys to nodes by hashing them onto a circular ring, enabling stable sharding of workloads across a dynamic set of nodes. When a node is added or removed, only the keys that fall in its ring segment are redistributed — minimizing data movement compared to naive modulo sharding. ExHashRing is used for routing work to worker pools, partitioning cache keys, and distributing load across dynamic node sets.

When ExHashRing's ring is stale or misconfigured, routing silently goes wrong. Work intended for node A goes to node B. Cache lookups miss because the key was hashed to a node that is no longer in the ring. Adding a node doesn't reduce load because the ring hasn't been updated. None of these failures produce errors — the wrong node simply handles the request.

Vigilmon heartbeat monitors let you verify that your ExHashRing is current, balanced, and routing correctly, not just that the ETS table exists.

Why Monitor ExHashRing?

ExHashRing failures are routing errors that look like application errors:

• Stale ring after node change — if you don't update the ring when nodes join or leave, keys continue routing to dead nodes; requests silently fail or go to the wrong partition
• Unbalanced virtual node distribution — if nodes use different virtual node counts, some nodes carry proportionally more keys; this is intentional for weighted rings but becomes a problem if it's accidental
• Ring state divergence across nodes — each application node maintains its own ring; if they aren't updated with the same membership list, different nodes route the same key to different targets
• High redistribution on membership changes — if virtual node count is too low, ring segments are large and node changes redistribute many keys; this can cause cache stampedes or job pile-ups on the new node
• Ring lookup performance degradation — ExHashRing uses ETS for fast ring lookups; if the ETS table is deleted or corrupted, lookups fall back to slower paths or fail

Key Metrics to Track

| Metric | What it tells you | |--------|-------------------| | Ring node count | Whether the ring reflects the current cluster membership | | Virtual node count per node | Whether virtual node distribution is balanced | | Key redistribution on node change | How many keys move when membership changes (lower is better) | | Ring lookup latency | ETS lookup performance; degradation indicates table pressure | | Routing target consistency | Whether all application nodes route the same key to the same target | | Ring generation / version | Whether all nodes have applied the same membership update |

Step 1: Add ExHashRing to Your Application

# mix.exs
defp deps do
  [
    {:ex_hash_ring, "~> 6.0"}
  ]
end

# lib/my_app/ring_manager.ex
defmodule MyApp.RingManager do
  use GenServer

  @virtual_nodes 128
  @ring_name :my_app_ring

  def start_link(_), do: GenServer.start_link(__MODULE__, %{}, name: __MODULE__)

  def init(state) do
    nodes = Application.get_env(:my_app, :ring_nodes, [])
    {:ok, ring} = ExHashRing.new(nodes, @virtual_nodes)
    :persistent_term.put(@ring_name, ring)
    {:ok, %{ring: ring, nodes: nodes, version: 0}}
  end

  def get_node(key) do
    ring = :persistent_term.get(@ring_name)
    ExHashRing.find_node(ring, key)
  end

  def update_nodes(nodes) do
    GenServer.call(__MODULE__, {:update_nodes, nodes})
  end

  def handle_call({:update_nodes, nodes}, _from, state) do
    {:ok, ring} = ExHashRing.new(nodes, @virtual_nodes)
    :persistent_term.put(@ring_name, ring)
    {:reply, :ok, %{state | ring: ring, nodes: nodes, version: state.version + 1}}
  end

  def get_state do
    GenServer.call(__MODULE__, :get_state)
  end

  def handle_call(:get_state, _from, state), do: {:reply, state, state}
end

Step 2: Build a Ring Health Check

# lib/my_app/ring_health.ex
defmodule MyApp.RingHealth do
  @probe_keys ["__health_probe_1__", "__health_probe_2__", "__health_probe_3__"]
  @expected_nodes Application.compile_env(:my_app, :ring_nodes, [])

  def check do
    pid = Process.whereis(MyApp.RingManager)

    if is_nil(pid) do
      %{status: :error, reason: "RingManager process not running"}
    else
      state = MyApp.RingManager.get_state()
      ring = :persistent_term.get(:my_app_ring, nil)

      if is_nil(ring) do
        %{status: :error, reason: "Ring not initialized in persistent_term"}
      else
        node_count = length(state.nodes)
        expected_count = length(@expected_nodes)
        probe_results = probe_routing(ring)

        status =
          cond do
            node_count == 0 -> :error
            node_count < expected_count -> :degraded
            Enum.any?(probe_results, fn {_, r} -> r == :error end) -> :degraded
            true -> :ok
          end

        %{
          status: status,
          ring_pid: inspect(pid),
          node_count: node_count,
          expected_node_count: expected_count,
          ring_version: state.version,
          nodes: state.nodes,
          probe_routing: probe_results
        }
      end
    end
  rescue
    e -> %{status: :error, reason: inspect(e)}
  end

  defp probe_routing(ring) do
    Enum.map(@probe_keys, fn key ->
      case ExHashRing.find_node(ring, key) do
        {:ok, node} -> {key, node}
        {:error, _} -> {key, :error}
      end
    end)
  end
end

Step 3: Check Cross-Node Ring Consistency

Verify all nodes route the same key to the same target:

# lib/my_app/ring_consistency.ex
defmodule MyApp.RingConsistency do
  @probe_keys ["consistency_probe_alpha", "consistency_probe_beta", "consistency_probe_gamma"]

  def check do
    cluster_nodes = [Node.self() | Node.list()]

    if length(cluster_nodes) < 2 do
      {:ok, :single_node}
    else
      local_routes = route_probes_locally()

      remote_routes =
        Enum.map(Node.list(), fn node ->
          case :rpc.call(node, MyApp.RingConsistency, :route_probes_locally, [], 3_000) do
            {:badrpc, reason} -> {node, {:error, reason}}
            routes -> {node, routes}
          end
        end)

      errors = Enum.filter(remote_routes, fn {_, r} -> match?({:error, _}, r) end)
      mismatches =
        remote_routes
        |> Enum.reject(fn {_, r} -> match?({:error, _}, r) end)
        |> Enum.filter(fn {_node, routes} -> routes != local_routes end)

      cond do
        errors != [] ->
          {:degraded, %{reason: :rpc_failures, errors: length(errors)}}

        mismatches != [] ->
          {:degraded, %{
            reason: :routing_mismatch,
            mismatched_nodes: Enum.map(mismatches, fn {n, _} -> to_string(n) end),
            local: local_routes
          }}

        true ->
          {:ok, %{nodes_checked: length(cluster_nodes), all_consistent: true}}
      end
    end
  end

  def route_probes_locally do
    ring = :persistent_term.get(:my_app_ring)

    Enum.map(@probe_keys, fn key ->
      case ExHashRing.find_node(ring, key) do
        {:ok, node} -> {key, to_string(node)}
        _ -> {key, :error}
      end
    end)
  end
end

Step 4: Expose a Health Endpoint

# lib/my_app_web/controllers/health_controller.ex
defmodule MyAppWeb.HealthController do
  use MyAppWeb, :controller

  def index(conn, _params) do
    ring = MyApp.RingHealth.check()
    consistency = MyApp.RingConsistency.check()

    ring_ok = ring.status == :ok
    consistency_ok = elem(consistency, 0) == :ok

    status = if ring_ok and consistency_ok, do: 200, else: 503

    conn
    |> put_status(status)
    |> json(%{
      status: if(status == 200, do: "ok", else: "degraded"),
      node: Node.self() |> to_string(),
      ring: ring,
      consistency: format_consistency(consistency)
    })
  end

  defp format_consistency({:ok, :single_node}), do: %{status: "ok", mode: "single_node"}
  defp format_consistency({:ok, data}), do: Map.merge(%{status: "ok"}, data)
  defp format_consistency({:degraded, data}), do: Map.merge(%{status: "degraded"}, data)
  defp format_consistency({:error, data}), do: Map.merge(%{status: "error"}, data)
end

# lib/my_app_web/router.ex
scope "/", MyAppWeb do
  get "/health", HealthController, :index
end

Step 5: Emit Telemetry Metrics

# lib/my_app/ring_poller.ex
defmodule MyApp.RingPoller do
  use GenServer
  require Logger

  @interval :timer.seconds(30)

  def start_link(_), do: GenServer.start_link(__MODULE__, %{}, name: __MODULE__)

  def init(state) do
    schedule()
    {:ok, state}
  end

  def handle_info(:poll, state) do
    report()
    schedule()
    {:noreply, state}
  end

  defp report do
    pid = Process.whereis(MyApp.RingManager)

    if pid do
      ring_state = MyApp.RingManager.get_state()
      node_count = length(ring_state.nodes)

      # Measure lookup latency
      ring = :persistent_term.get(:my_app_ring, nil)
      {lookup_us, _result} = if ring, do: :timer.tc(fn -> ExHashRing.find_node(ring, "benchmark_key") end), else: {0, nil}

      :telemetry.execute(
        [:my_app, :ring, :state],
        %{node_count: node_count, version: ring_state.version, lookup_latency_us: lookup_us},
        %{node: Node.self()}
      )
    else
      Logger.warning("RingPoller: RingManager process not running")
    end
  rescue
    e -> Logger.warning("RingPoller error: #{inspect(e)}")
  end

  defp schedule, do: Process.send_after(self(), :poll, @interval)
end

Step 6: Create Monitors in Vigilmon

HTTP monitor for the health endpoint:

1. Sign in at vigilmon.online
2. Click New Monitor → HTTP
3. URL: https://your-app.example.com/health
4. Interval: 60 seconds
5. Alert condition: non-200 or body contains "ring":{"status":"degraded"}

Heartbeat monitor for ring liveness:

# lib/my_app/ring_heartbeat.ex
defmodule MyApp.RingHeartbeat do
  use GenServer

  @interval :timer.minutes(2)

  def start_link(_), do: GenServer.start_link(__MODULE__, %{}, name: __MODULE__)

  def init(state) do
    schedule()
    {:ok, state}
  end

  def handle_info(:ping, state) do
    check = MyApp.RingHealth.check()

    if check.status == :ok do
      url = System.get_env("VIGILMON_RING_HEARTBEAT_URL")

      if url do
        :httpc.request(:get, {String.to_charlist(url), []}, [{:timeout, 5_000}], [])
      end
    end

    schedule()
    {:noreply, state}
  end

  defp schedule, do: Process.send_after(self(), :ping, @interval)
end

In Vigilmon, create a Heartbeat monitor with a 5-minute expected interval. Set VIGILMON_RING_HEARTBEAT_URL from the Vigilmon dashboard.

Step 7: Alerting

In Vigilmon, go to Notifications → New Channel:

Slack for ring degradation:

🔴 DOWN: ExHashRing — MyApp
Monitor: /health returning ring.status = "degraded"
Node: myapp@10.0.0.1
Action: Check RingManager process, node membership list, and cluster connectivity

PagerDuty for heartbeat miss — a missed ring heartbeat means key routing is broken or the RingManager is down; all sharding decisions are now incorrect.

Common ExHashRing Issues and Fixes

Stale ring after node failure:

# Integrate cluster membership callbacks to auto-update the ring
# Using libcluster, subscribe to :nodeup/:nodedown events:
def handle_info({:nodeup, node}, state) do
  new_nodes = [node | state.nodes] |> Enum.uniq()
  MyApp.RingManager.update_nodes(new_nodes)
  {:noreply, %{state | nodes: new_nodes}}
end

def handle_info({:nodedown, node}, state) do
  new_nodes = List.delete(state.nodes, node)
  MyApp.RingManager.update_nodes(new_nodes)
  {:noreply, %{state | nodes: new_nodes}}
end

Ring routing mismatch across nodes:

# From each node's IEx, route the same probe key
iex(node_a@host)> ring = :persistent_term.get(:my_app_ring)
iex(node_a@host)> ExHashRing.find_node(ring, "user:12345")
{:ok, "worker_pool_3"}

iex(node_b@host)> ring = :persistent_term.get(:my_app_ring)
iex(node_b@host)> ExHashRing.find_node(ring, "user:12345")
{:ok, "worker_pool_1"}  # <-- mismatch; node_b has a different ring
# Trigger ring update on node_b:
iex(node_b@host)> MyApp.RingManager.update_nodes(["worker_pool_1", "worker_pool_2", "worker_pool_3"])

High redistribution on node add:

# Increase virtual node count to reduce average segment size
# Higher virtual nodes = more even distribution = less redistribution per membership change
# Tune based on your node count: 128-512 virtual nodes is typical
{:ok, ring} = ExHashRing.new(nodes, 256)  # was 128; double for smoother redistribution

What You've Built

| What | How | |------|-----| | Ring process check | Verify RingManager alive and probe key routing works | | Cross-node routing consistency | RPC comparison of routing targets for probe keys | | Structured health endpoint | JSON health response with ring and consistency status | | Ring metrics | Telemetry on node count, version, and lookup latency every 30s | | Liveness heartbeat | GenServer pinging Vigilmon when ring is healthy | | Real-time alerting | Vigilmon HTTP monitor + PagerDuty on heartbeat miss |

ExHashRing makes stable key-to-node routing possible across a dynamic cluster. Vigilmon makes sure the ring is current and consistent when routing decisions actually matter.

Next Steps

• Emit metrics on redistribution count when ring membership changes to quantify cache churn per node addition
• Add per-application-domain rings if you shard cache keys and work queues separately
• Alert when node count drops below your expected minimum to catch silent node loss before routing degrades
• Use Vigilmon's downtime history to correlate ring inconsistency events with application error rate spikes

Get started free at vigilmon.online.