handle errors from cron logs

Author: vrtnd

src/adapters/axelar-satellite/index.ts

@@ -19,18 +19,26 @@ import {
   throwIfAborted,
   waitWithSignal,
 } from "../../utils/errors";
+import { ConsecutiveFailureCircuitBreaker } from "../../helpers/circuitBreaker";
 const retry = require("async-retry");
 
 const AXELAR_API_URL = "https://api.axelarscan.io/";
 const AXELAR_REQUEST_TIMEOUT_MS = 30_000;
 const AXELAR_REQUEST_INTERVAL_MS = 350;
 const AXELAR_REQUEST_CONCURRENCY = 3;
 const AXELAR_REQUEST_RETRIES = 3;
+const AXELAR_CIRCUIT_FAILURE_THRESHOLD = 3;
+const AXELAR_CIRCUIT_COOLDOWN_MS = 10 * 60 * 1000;
 
 let activeAxelarRequests = 0;
 let nextAxelarRequestAt = 0;
 const axelarWaiters: Array<() => void> = [];
 let assetsPromise: Promise<any[]> | undefined;
+const axelarCircuit = new ConsecutiveFailureCircuitBreaker(
+  "Axelarscan",
+  AXELAR_CIRCUIT_FAILURE_THRESHOLD,
+  AXELAR_CIRCUIT_COOLDOWN_MS
+);
 
 const acquireAxelarSlot = async (signal?: AbortSignal) => {
   throwIfAborted(signal);
@@ -82,8 +90,10 @@ const fetchAxelar = async <T>(body: Record<string, unknown>, signal?: AbortSigna
       let acquired = false;
       try {
         throwIfAborted(signal);
+        axelarCircuit.assertAvailable();
         await acquireAxelarSlot(signal);
         acquired = true;
+        axelarCircuit.assertAvailable();
         const response = await fetch(AXELAR_API_URL, {
           method: "POST",
           headers: {
@@ -95,13 +105,21 @@ const fetchAxelar = async <T>(body: Record<string, unknown>, signal?: AbortSigna
         });
         if (!response.ok) {
           const responseBody = await response.text().catch(() => "");
-          const error =
-            response.status !== 429 && response.status < 500
-              ? new NonRetryableError(`Axelarscan HTTP ${response.status}: ${responseBody.slice(0, 300)}`)
-              : new Error(`Axelarscan HTTP ${response.status}: ${responseBody.slice(0, 300)}`);
+          const message = `Axelarscan HTTP ${response.status}: ${responseBody.slice(0, 300)}`;
+          let error: Error;
+          if (response.status >= 500 && axelarCircuit.recordFailure()) {
+            error = new NonRetryableError(
+              `${message}; circuit opened after ${AXELAR_CIRCUIT_FAILURE_THRESHOLD} consecutive server failures.`
+            );
+          } else {
+            error =
+              response.status !== 429 && response.status < 500 ? new NonRetryableError(message) : new Error(message);
+          }
           throw error;
         }
-        return (await response.json()) as T;
+        const result = (await response.json()) as T;
+        axelarCircuit.recordSuccess();
+        return result;
       } catch (error) {
         if (isAbortError(error) || signal?.aborted || isNonRetryableError(error)) return bail(error as Error);
         throw error;

src/adapters/axelar/index.ts

@@ -243,7 +243,7 @@ const InterchainTransferReceivedParams: PartialContractEventParams = {
   isDeposit: false,
 };
 
-const constructParams = (chain: SupportedChains) => {
+const constructParams = (chain: SupportedChains, providerChain: string = chain) => {
   const axelarContractAddress = axelarChains[chain];
 
   const gatewayAddy = axelarContractAddress.gateway;
@@ -307,7 +307,7 @@ const constructParams = (chain: SupportedChains) => {
     const interchainTransferSentEvent = {
       ...InterchainTransferSentParams,
       target: itsAddy,
-      chain: chain,
+      chain: providerChain,
       argGetters: {
         amount: (log: any) => BigNumber.from(log.amount),
         to: (log: any) => log.destinationAddress,
@@ -324,7 +324,7 @@ const constructParams = (chain: SupportedChains) => {
     const interchainTransferReceivedEvent = {
       ...InterchainTransferReceivedParams,
       target: itsAddy,
-      chain: chain,
+      chain: providerChain,
       argGetters: {
         amount: (log: any) => BigNumber.from(log.amount),
         to: (log: any) => log.destinationAddress,
@@ -348,7 +348,7 @@ const constructParams = (chain: SupportedChains) => {
       interchainTransferSentEvent,
       interchainTransferReceivedEvent
     );
-    return await getTxDataFromEVMEventLogs("axelar", chain, fromBlock, toBlock, eventParams);
+    return await getTxDataFromEVMEventLogs("axelar", providerChain as any, fromBlock, toBlock, eventParams);
   };
 };
 const adapter: BridgeAdapter = {
@@ -370,7 +370,7 @@ const adapter: BridgeAdapter = {
   mantle: constructParams("mantle"),
   moonbeam: constructParams("moonbeam"),
   optimism: constructParams("optimism"),
-  plume: constructParams("plume"),
+  plume: constructParams("plume", "plume_mainnet"),
   polygon: constructParams("polygon"),
   scroll: constructParams("scroll"),
   monad: constructParams("monad"),

src/adapters/debridgedln/index.ts

@@ -79,7 +79,7 @@ const withdrawParams: PartialContractEventParams = {
   isDeposit: false,
 };
 
-const constructParams = (chain: SupportedChains) => {
+const constructParams = (chain: SupportedChains, providerChain: string = chain) => {
   const eventParams: PartialContractEventParams[] = [];
 
   const token = nativeTokenAddress[chain];
@@ -97,7 +97,7 @@ const constructParams = (chain: SupportedChains) => {
   eventParams.push(finalDepositParams, finalWithdrawParams);
 
   return async (fromBlock: number, toBlock: number) =>
-    getTxDataFromEVMEventLogs("debridgedln", chain, fromBlock, toBlock, eventParams);
+    getTxDataFromEVMEventLogs("debridgedln", providerChain as any, fromBlock, toBlock, eventParams);
 };
 
 type ApiSolanaEvent = {
@@ -158,7 +158,7 @@ const adapter: BridgeAdapter = {
   solana: getSolanaEvents,
   sonic: constructParams("sonic"),
   plasma: constructParams("plasma"),
-  plume: constructParams("plume"),
+  plume: constructParams("plume", "plume_mainnet"),
   monad: constructParams("monad"),
   robinhood: constructParams("robinhood"),
 };

src/adapters/hyperlane/index.ts

@@ -4,6 +4,7 @@ import { Chain } from "@defillama/sdk/build/general";
 import * as yaml from "js-yaml";
 
 import { BridgeAdapter } from "../../helpers/bridgeAdapter.type";
+import { NonRetryableError } from "../../utils/errors";
 
 const baseUri = "https://raw.githubusercontent.com/hyperlane-xyz/hyperlane-registry/main";
 const kyveApiBaseUri = "https://data.services.hyperlane.xyz";
@@ -24,7 +25,7 @@ export async function setUp(): Promise<string[]> {
   return chains;
 }
 
-interface KyveEvent {
+export interface KyveEvent {
   blockNumber: number;
   chain: string;
   from: string;
@@ -36,47 +37,75 @@ interface KyveEvent {
   usdAmount: string;
 }
 
-export const getEvents = async (fromTimestamp: number, toTimestamp: number): Promise<any[]> => {
+type HyperlaneFetch = (
+  input: string,
+  init?: { headers?: Record<string, string> }
+) => Promise<{
+  ok: boolean;
+  status: number;
+  statusText: string;
+  text(): Promise<string>;
+  json(): Promise<unknown>;
+}>;
+
+export const fetchHyperlaneEvents = async (
+  fromTimestamp: number,
+  toTimestamp: number,
+  fetchImpl: HyperlaneFetch = fetch as HyperlaneFetch
+): Promise<KyveEvent[]> => {
   const apiUrl = `${kyveApiBaseUri}/events?fromTimestamp=${fromTimestamp}&toTimestamp=${toTimestamp}`;
   console.log(apiUrl);
-  try {
-    const response = await fetch(apiUrl);
-    if (!response.ok) {
-      console.error(`Error fetching data from Kyve API: ${response.statusText}`);
-      return [];
+  const response = await fetchImpl(apiUrl, {
+    headers: {
+      Accept: "application/json",
+      "User-Agent": "defillama-bridges-server/1.0",
+    },
+  });
+  if (!response.ok) {
+    const responseBody = await response.text().catch(() => "");
+    const detail = responseBody.trim().slice(0, 300);
+    const message = `Hyperlane events API HTTP ${response.status} ${response.statusText}${detail ? `: ${detail}` : ""}`;
+    if (response.status >= 400 && response.status < 500 && response.status !== 429) {
+      throw new NonRetryableError(message);
     }
-    const events = (await response.json()) as KyveEvent[];
-
-    const txData: any[] = events.map((event) => {
-      let usdAmount: number | undefined = undefined;
-      try {
-        usdAmount = parseFloat(event.usdAmount);
-        if (isNaN(usdAmount)) {
-          usdAmount = undefined;
-        }
-      } catch {
+    throw new Error(message);
+  }
+
+  const events = await response.json();
+  if (!Array.isArray(events)) {
+    throw new NonRetryableError("Hyperlane events API returned a non-array response.");
+  }
+  return events as KyveEvent[];
+};
+
+export const getEvents = async (fromTimestamp: number, toTimestamp: number): Promise<any[]> => {
+  const events = await fetchHyperlaneEvents(fromTimestamp, toTimestamp);
+  const txData: any[] = events.map((event) => {
+    let usdAmount: number | undefined = undefined;
+    try {
+      usdAmount = parseFloat(event.usdAmount);
+      if (isNaN(usdAmount)) {
         usdAmount = undefined;
       }
+    } catch {
+      usdAmount = undefined;
+    }
 
-      return {
-        blockNumber: event.blockNumber,
-        chain: event.chain,
-        from: event.from,
-        isDeposit: event.isDeposit,
-        timestamp: event.timestamp,
-        to: event.to,
-        token: event.token,
-        txHash: event.txHash,
-        amount: usdAmount,
-        isUSDVolume: true,
-      };
-    });
-
-    return txData;
-  } catch (error) {
-    console.error(`Error processing Kyve API response:`, error);
-    return [];
-  }
+    return {
+      blockNumber: event.blockNumber,
+      chain: event.chain,
+      from: event.from,
+      isDeposit: event.isDeposit,
+      timestamp: event.timestamp,
+      to: event.to,
+      token: event.token,
+      txHash: event.txHash,
+      amount: usdAmount,
+      isUSDVolume: true,
+    };
+  });
+
+  return txData;
 };
 
 export async function build(): Promise<BridgeAdapter> {

src/adapters/relay/index.ts

@@ -361,7 +361,10 @@ export const slugToChainId: Record<string, number> = {
   "polygon-zkevm": 1101,
   lisk: 1135,
   sei: 1329,
+  hyperliquid: 1337,
   hychain: 2911,
+  ronin: 2020,
+  somnia: 5031,
   mantle: 5000,
   ham: 5112,
   cyber: 7560,

src/handlers/relayApi.test.ts

@@ -1,6 +1,6 @@
 import assert from "node:assert/strict";
 import test from "node:test";
-import { makeRequestsUrl, parseRelayRequestsResponse } from "../adapters/relay";
+import { makeRequestsUrl, parseRelayRequestsResponse, slugToChainId } from "../adapters/relay";
 
 test("Relay windows are filtered and sorted by updatedAt", () => {
   const url = new URL(makeRequestsUrl(100, 200, "next", 1));
@@ -12,6 +12,12 @@ test("Relay windows are filtered and sorted by updatedAt", () => {
   assert.equal(url.searchParams.get("chainId"), "1");
 });
 
+test("Relay maps currently supported non-EVM and emerging chain IDs", () => {
+  assert.equal(slugToChainId.hyperliquid, 1337);
+  assert.equal(slugToChainId.ronin, 2020);
+  assert.equal(slugToChainId.somnia, 5031);
+});
+
 test("Relay rejects malformed successful responses before checkpoint advancement", () => {
   assert.throws(() => parseRelayRequestsResponse({}), /requests array/);
   assert.throws(() => parseRelayRequestsResponse({ requests: [], continuation: 123 }), /continuation/);

src/handlers/relayProgress.test.ts

@@ -1,6 +1,6 @@
 import assert from "node:assert/strict";
 import test from "node:test";
-import { requireRelayChainId, resolveRelayWindowFromCheckpoint } from "./relayProgress";
+import { getRelayBootstrapCheckpoint, requireRelayChainId, resolveRelayWindowFromCheckpoint } from "./relayProgress";
 
 const defaults = {
   now: 10_000,
@@ -29,6 +29,31 @@ test("Relay bootstraps from the configured lookback when the Redis key is missin
   });
 });
 
+test("Relay persists a stable bootstrap low-water mark across failed hourly runs", () => {
+  const checkpoint = getRelayBootstrapCheckpoint(defaults.now, defaults.bootstrapLookbackSeconds);
+  const firstRun = resolveRelayWindowFromCheckpoint({
+    ...defaults,
+    checkpoint,
+    source: "redis",
+  });
+  const nextHourlyRun = resolveRelayWindowFromCheckpoint({
+    ...defaults,
+    now: defaults.now + 3_600,
+    checkpoint,
+    source: "redis",
+  });
+
+  assert.equal(firstRun.startTs, 7_700);
+  assert.equal(nextHourlyRun.startTs, firstRun.startTs);
+  assert.equal(nextHourlyRun.endTs, defaults.now + 3_600);
+});
+
+test("Relay validates bootstrap checkpoint inputs", () => {
+  assert.equal(getRelayBootstrapCheckpoint(10_000, 2_000), 8_000);
+  assert.throws(() => getRelayBootstrapCheckpoint(0, 2_000), /positive Unix timestamp/);
+  assert.throws(() => getRelayBootstrapCheckpoint(10_000, 0), /positive integer/);
+});
+
 test("Relay fails closed with a far-future checkpoint", () => {
   assert.throws(
     () => resolveRelayWindowFromCheckpoint({ ...defaults, checkpoint: 10_061, source: "redis" }),

src/handlers/relayProgress.ts

@@ -3,6 +3,18 @@ import { NonRetryableError } from "../utils/errors";
 export type RelayCheckpointSource = "redis" | "bootstrap";
 export const RELAY_CHECKPOINT_MAX_FUTURE_SKEW_SECONDS = 60;
 
+export const getRelayBootstrapCheckpoint = (now: number, bootstrapLookbackSeconds: number): number => {
+  if (!Number.isSafeInteger(now) || now <= 0) {
+    throw new NonRetryableError(`Relay bootstrap time must be a positive Unix timestamp; received ${now}.`);
+  }
+  if (!Number.isSafeInteger(bootstrapLookbackSeconds) || bootstrapLookbackSeconds <= 0) {
+    throw new NonRetryableError(
+      `Relay bootstrap lookback must be a positive integer; received ${bootstrapLookbackSeconds}.`
+    );
+  }
+  return Math.max(1, now - bootstrapLookbackSeconds);
+};
+
 export const requireRelayChainId = (leg: "deposit" | "withdrawal", chainId?: number): number => {
   if (!Number.isInteger(chainId) || Number(chainId) <= 0) {
     throw new NonRetryableError(`Relay ${leg} is missing a valid chain ID`);

src/handlers/runRelay.ts

@@ -7,6 +7,7 @@ import dayjs from "dayjs";
 import { formatError, NonRetryableError, throwIfAborted } from "../utils/errors";
 import {
   RelayCheckpointSource,
+  getRelayBootstrapCheckpoint,
   requireRelayChainId,
   resolveRelayWindowFromCheckpoint,
   validateRelayCheckpoint,
@@ -81,9 +82,18 @@ const getLatestRelayCheckpoint = async (
   source: RelayCheckpointSource;
 }> => {
   const durableCheckpoint = await getDurableCheckpoint(RELAY_CHECKPOINT_KEY);
-  return durableCheckpoint !== null
-    ? { checkpoint: validateRelayCheckpoint(durableCheckpoint, now), source: "redis" }
-    : { checkpoint: null, source: "bootstrap" };
+  if (durableCheckpoint !== null) {
+    return { checkpoint: validateRelayCheckpoint(durableCheckpoint, now), source: "redis" };
+  }
+
+  const bootstrapCheckpoint = getRelayBootstrapCheckpoint(now, BOOTSTRAP_LOOKBACK_HOURS * 60 * 60);
+  const storedCheckpoint = await advanceDurableCheckpoint(RELAY_CHECKPOINT_KEY, bootstrapCheckpoint);
+  console.log(
+    `Relay bootstrap checkpoint initialized in Redis: ${dayjs
+      .unix(storedCheckpoint)
+      .toISOString()} (${RELAY_CHECKPOINT_KEY})`
+  );
+  return { checkpoint: validateRelayCheckpoint(storedCheckpoint, now), source: "redis" };
 };
 
 const resolveWindow = async () => {