openzeppelin_relayer/services/provider/

mod.rs

use std::num::ParseIntError;
use std::time::Duration;

use once_cell::sync::Lazy;
use reqwest::Client as ReqwestClient;
use tracing::debug;

use crate::config::ServerConfig;
use crate::constants::{
    matches_known_transaction, ALREADY_SUBMITTED_PATTERNS,
    DEFAULT_HTTP_CLIENT_CONNECT_TIMEOUT_SECONDS,
    DEFAULT_HTTP_CLIENT_HTTP2_KEEP_ALIVE_INTERVAL_SECONDS,
    DEFAULT_HTTP_CLIENT_HTTP2_KEEP_ALIVE_TIMEOUT_SECONDS,
    DEFAULT_HTTP_CLIENT_POOL_IDLE_TIMEOUT_SECONDS, DEFAULT_HTTP_CLIENT_POOL_MAX_IDLE_PER_HOST,
    DEFAULT_HTTP_CLIENT_TCP_KEEPALIVE_SECONDS, NONCE_TOO_HIGH_PATTERNS,
};
use crate::models::{EvmNetwork, RpcConfig, SolanaNetwork, StellarNetwork};
use crate::utils::create_secure_redirect_policy;
use serde::Serialize;
use thiserror::Error;

use alloy::transports::RpcError;

pub mod evm;
pub use evm::*;

mod solana;
pub use solana::*;

mod stellar;
pub use stellar::*;

mod retry;
pub use retry::*;

pub mod rpc_health_store;
pub mod rpc_selector;

pub use rpc_health_store::{RpcConfigMetadata, RpcHealthStore};

/// Configuration for creating a provider instance.
///
/// This struct encapsulates all the parameters needed to create a provider,
/// making the API cleaner and easier to maintain.
#[derive(Debug, Clone)]
pub struct ProviderConfig {
    /// RPC endpoint configurations (URLs and weights)
    pub rpc_configs: Vec<RpcConfig>,
    /// Timeout duration in seconds for RPC requests
    pub timeout_seconds: u64,
    /// Number of consecutive failures before pausing a provider
    pub failure_threshold: u32,
    /// Duration in seconds to pause a provider after reaching failure threshold
    pub pause_duration_secs: u64,
    /// Duration in seconds after which failures are considered stale and reset
    pub failure_expiration_secs: u64,
}

impl ProviderConfig {
    /// Creates a new `ProviderConfig` from individual parameters.
    ///
    /// # Arguments
    /// * `rpc_configs` - RPC endpoint configurations
    /// * `timeout_seconds` - Timeout duration in seconds
    /// * `failure_threshold` - Number of consecutive failures before pausing
    /// * `pause_duration_secs` - Duration in seconds to pause after threshold
    /// * `failure_expiration_secs` - Duration in seconds after which failures are considered stale
    pub fn new(
        rpc_configs: Vec<RpcConfig>,
        timeout_seconds: u64,
        failure_threshold: u32,
        pause_duration_secs: u64,
        failure_expiration_secs: u64,
    ) -> Self {
        Self {
            rpc_configs,
            timeout_seconds,
            failure_threshold,
            pause_duration_secs,
            failure_expiration_secs,
        }
    }

    /// Creates a `ProviderConfig` from `ServerConfig` with the given RPC configs.
    ///
    /// This is a convenience method that extracts provider-related configuration
    /// from the server configuration.
    ///
    /// # Arguments
    /// * `server_config` - The server configuration
    /// * `rpc_configs` - RPC endpoint configurations
    pub fn from_server_config(server_config: &ServerConfig, rpc_configs: Vec<RpcConfig>) -> Self {
        let timeout_seconds = server_config.rpc_timeout_ms / 1000; // Convert ms to s
        Self {
            rpc_configs,
            timeout_seconds,
            failure_threshold: server_config.provider_failure_threshold,
            pause_duration_secs: server_config.provider_pause_duration_secs,
            failure_expiration_secs: server_config.provider_failure_expiration_secs,
        }
    }

    /// Creates a `ProviderConfig` from environment variables with the given RPC configs.
    ///
    /// This loads configuration from `ServerConfig::from_env()`.
    ///
    /// # Arguments
    /// * `rpc_configs` - RPC endpoint configurations
    pub fn from_env(rpc_configs: Vec<RpcConfig>) -> Self {
        let server_config = ServerConfig::from_env();
        Self::from_server_config(&server_config, rpc_configs)
    }
}

/// Pre-configured `reqwest::ClientBuilder` with standard pool, keepalive, TLS,
/// and redirect settings. Callers chain on extras (e.g., `.timeout(...)`) then `.build()`.
fn base_rpc_client_builder() -> reqwest::ClientBuilder {
    ReqwestClient::builder()
        .connect_timeout(Duration::from_secs(
            DEFAULT_HTTP_CLIENT_CONNECT_TIMEOUT_SECONDS,
        ))
        .pool_max_idle_per_host(DEFAULT_HTTP_CLIENT_POOL_MAX_IDLE_PER_HOST)
        .pool_idle_timeout(Duration::from_secs(
            DEFAULT_HTTP_CLIENT_POOL_IDLE_TIMEOUT_SECONDS,
        ))
        .tcp_keepalive(Duration::from_secs(
            DEFAULT_HTTP_CLIENT_TCP_KEEPALIVE_SECONDS,
        ))
        .http2_keep_alive_interval(Some(Duration::from_secs(
            DEFAULT_HTTP_CLIENT_HTTP2_KEEP_ALIVE_INTERVAL_SECONDS,
        )))
        .http2_keep_alive_timeout(Duration::from_secs(
            DEFAULT_HTTP_CLIENT_HTTP2_KEEP_ALIVE_TIMEOUT_SECONDS,
        ))
        .use_rustls_tls()
        .redirect(create_secure_redirect_policy())
}

/// Shared `reqwest::Client` for RPC providers that set per-request timeouts
/// (e.g., Stellar raw HTTP). No request-level timeout is baked in.
static SHARED_RPC_HTTP_CLIENT: Lazy<Result<ReqwestClient, String>> = Lazy::new(|| {
    debug!("Creating shared RPC HTTP client");
    base_rpc_client_builder()
        .build()
        .map_err(|e| format!("Failed to create shared RPC HTTP client: {e}"))
});

/// Get the shared RPC HTTP client (no per-request timeout).
pub fn get_shared_rpc_http_client() -> Result<ReqwestClient, ProviderError> {
    SHARED_RPC_HTTP_CLIENT
        .as_ref()
        .map(|c| c.clone())
        .map_err(|e| ProviderError::NetworkConfiguration(e.clone()))
}

/// Build a new RPC HTTP client with standard settings plus a per-request timeout.
/// Use when the provider needs timeouts baked into the client (e.g., EVM via alloy transport).
pub fn build_rpc_http_client_with_timeout(
    timeout: Duration,
) -> Result<ReqwestClient, ProviderError> {
    base_rpc_client_builder()
        .timeout(timeout)
        .build()
        .map_err(|e| {
            ProviderError::NetworkConfiguration(format!("Failed to build RPC HTTP client: {e}"))
        })
}

#[derive(Error, Debug, Serialize)]
pub enum ProviderError {
    #[error("RPC client error: {0}")]
    SolanaRpcError(#[from] SolanaProviderError),
    #[error("Invalid address: {0}")]
    InvalidAddress(String),
    #[error("Network configuration error: {0}")]
    NetworkConfiguration(String),
    #[error("Request timeout")]
    Timeout,
    #[error("Rate limited (HTTP 429)")]
    RateLimited,
    #[error("Bad gateway (HTTP 502)")]
    BadGateway,
    #[error("Request error (HTTP {status_code}): {error}")]
    RequestError { error: String, status_code: u16 },
    #[error("JSON-RPC error (code {code}): {message}")]
    RpcErrorCode { code: i64, message: String },
    #[error("Transport error: {0}")]
    TransportError(String),
    #[error("Other provider error: {0}")]
    Other(String),
}

impl ProviderError {
    /// Determines if this error is transient (can retry) or permanent (should fail).
    pub fn is_transient(&self) -> bool {
        is_retriable_error(self)
    }
}

impl From<hex::FromHexError> for ProviderError {
    fn from(err: hex::FromHexError) -> Self {
        ProviderError::InvalidAddress(err.to_string())
    }
}

impl From<std::net::AddrParseError> for ProviderError {
    fn from(err: std::net::AddrParseError) -> Self {
        ProviderError::NetworkConfiguration(format!("Invalid network address: {err}"))
    }
}

impl From<ParseIntError> for ProviderError {
    fn from(err: ParseIntError) -> Self {
        ProviderError::Other(format!("Number parsing error: {err}"))
    }
}

/// Categorizes a reqwest error into an appropriate `ProviderError` variant.
///
/// This function analyzes the given reqwest error and maps it to a specific
/// `ProviderError` variant based on the error's properties:
/// - Timeout errors become `ProviderError::Timeout`
/// - HTTP 429 responses become `ProviderError::RateLimited`
/// - HTTP 502 responses become `ProviderError::BadGateway`
/// - All other errors become `ProviderError::Other` with the error message
///
/// # Arguments
///
/// * `err` - A reference to the reqwest error to categorize
///
/// # Returns
///
/// The appropriate `ProviderError` variant based on the error type
fn categorize_reqwest_error(err: &reqwest::Error) -> ProviderError {
    if err.is_timeout() {
        return ProviderError::Timeout;
    }

    if let Some(status) = err.status() {
        match status.as_u16() {
            429 => return ProviderError::RateLimited,
            502 => return ProviderError::BadGateway,
            _ => {
                return ProviderError::RequestError {
                    error: err.to_string(),
                    status_code: status.as_u16(),
                }
            }
        }
    }

    ProviderError::Other(err.to_string())
}

impl From<reqwest::Error> for ProviderError {
    fn from(err: reqwest::Error) -> Self {
        categorize_reqwest_error(&err)
    }
}

impl From<&reqwest::Error> for ProviderError {
    fn from(err: &reqwest::Error) -> Self {
        categorize_reqwest_error(err)
    }
}

impl From<eyre::Report> for ProviderError {
    fn from(err: eyre::Report) -> Self {
        // Downcast to known error types first
        if let Some(reqwest_err) = err.downcast_ref::<reqwest::Error>() {
            return ProviderError::from(reqwest_err);
        }

        // Default to Other for unknown error types
        ProviderError::Other(err.to_string())
    }
}

// Add conversion from String to ProviderError
impl From<String> for ProviderError {
    fn from(error: String) -> Self {
        ProviderError::Other(error)
    }
}

// Generic implementation for all RpcError types
impl<E> From<RpcError<E>> for ProviderError
where
    E: std::fmt::Display + std::any::Any + 'static,
{
    fn from(err: RpcError<E>) -> Self {
        match err {
            RpcError::Transport(transport_err) => {
                // First check if it's a reqwest::Error using downcasting
                if let Some(reqwest_err) =
                    (&transport_err as &dyn std::any::Any).downcast_ref::<reqwest::Error>()
                {
                    return categorize_reqwest_error(reqwest_err);
                }

                ProviderError::TransportError(transport_err.to_string())
            }
            RpcError::ErrorResp(json_rpc_err) => ProviderError::RpcErrorCode {
                code: json_rpc_err.code,
                message: json_rpc_err.message.to_string(),
            },
            _ => ProviderError::Other(format!("Other RPC error: {err}")),
        }
    }
}

// Implement From for RpcSelectorError
impl From<rpc_selector::RpcSelectorError> for ProviderError {
    fn from(err: rpc_selector::RpcSelectorError) -> Self {
        ProviderError::NetworkConfiguration(format!("RPC selector error: {err}"))
    }
}

pub trait NetworkConfiguration: Sized {
    type Provider;

    fn public_rpc_urls(&self) -> Vec<RpcConfig>;

    /// Creates a new provider instance using the provided configuration.
    ///
    /// # Arguments
    /// * `config` - Provider configuration containing RPC configs and settings
    fn new_provider(config: ProviderConfig) -> Result<Self::Provider, ProviderError>;
}

impl NetworkConfiguration for EvmNetwork {
    type Provider = EvmProvider;

    fn public_rpc_urls(&self) -> Vec<RpcConfig> {
        self.rpc_urls.clone()
    }

    fn new_provider(config: ProviderConfig) -> Result<Self::Provider, ProviderError> {
        EvmProvider::new(config)
    }
}

impl NetworkConfiguration for SolanaNetwork {
    type Provider = SolanaProvider;

    fn public_rpc_urls(&self) -> Vec<RpcConfig> {
        self.rpc_urls.clone()
    }

    fn new_provider(config: ProviderConfig) -> Result<Self::Provider, ProviderError> {
        SolanaProvider::new(config)
    }
}

impl NetworkConfiguration for StellarNetwork {
    type Provider = StellarProvider;

    fn public_rpc_urls(&self) -> Vec<RpcConfig> {
        self.rpc_urls.clone()
    }

    fn new_provider(config: ProviderConfig) -> Result<Self::Provider, ProviderError> {
        StellarProvider::new(config)
    }
}

/// Creates a network-specific provider instance based on the provided configuration.
///
/// # Type Parameters
///
/// * `N`: The type of the network, which must implement the `NetworkConfiguration` trait.
///   This determines the specific provider type (`N::Provider`) and how to obtain
///   public RPC URLs.
///
/// # Arguments
///
/// * `network`: A reference to the network configuration object (`&N`).
/// * `custom_rpc_urls`: An `Option<Vec<RpcConfig>>`. If `Some` and not empty, these URLs
///   are used to configure the provider. If `None` or `Some` but empty, the function
///   falls back to using the public RPC URLs defined by the `network`'s
///   `NetworkConfiguration` implementation.
///
/// # Returns
///
/// * `Ok(N::Provider)`: An instance of the network-specific provider on success.
/// * `Err(ProviderError)`: An error if configuration fails, such as when no custom URLs
///   are provided and the network has no public RPC URLs defined
///   (`ProviderError::NetworkConfiguration`).
pub fn get_network_provider<N: NetworkConfiguration>(
    network: &N,
    custom_rpc_urls: Option<Vec<RpcConfig>>,
) -> Result<N::Provider, ProviderError> {
    let rpc_urls = match custom_rpc_urls {
        Some(configs) if !configs.is_empty() => configs,
        _ => {
            let configs = network.public_rpc_urls();
            if configs.is_empty() {
                return Err(ProviderError::NetworkConfiguration(
                    "No public RPC URLs available for this network".to_string(),
                ));
            }
            configs
        }
    };

    let provider_config = ProviderConfig::from_env(rpc_urls);
    N::new_provider(provider_config)
}

/// Determines if an HTTP status code indicates the provider should be marked as failed.
///
/// This is a low-level function that can be reused across different error types.
///
/// Returns `true` for:
/// - 5xx Server Errors (500-599) - RPC node is having issues
/// - Specific 4xx Client Errors that indicate provider issues:
///   - 401 (Unauthorized) - auth required but not provided
///   - 403 (Forbidden) - node is blocking requests or auth issues
///   - 404 (Not Found) - endpoint doesn't exist or misconfigured
///   - 410 (Gone) - endpoint permanently removed
pub fn should_mark_provider_failed_by_status_code(status_code: u16) -> bool {
    match status_code {
        // 5xx Server Errors - RPC node is having issues
        500..=599 => true,

        // 4xx Client Errors that indicate we can't use this provider
        401 => true, // Unauthorized - auth required but not provided
        403 => true, // Forbidden - node is blocking requests or auth issues
        404 => true, // Not Found - endpoint doesn't exist or misconfigured
        410 => true, // Gone - endpoint permanently removed

        _ => false,
    }
}

pub fn should_mark_provider_failed(error: &ProviderError) -> bool {
    match error {
        ProviderError::RequestError { status_code, .. } => {
            should_mark_provider_failed_by_status_code(*status_code)
        }
        _ => false,
    }
}

/// Returns true if the RPC error message indicates a transaction-level error
/// that should not be retried — the RPC is working correctly, but rejecting
/// the transaction itself.
///
/// Uses the shared `ALREADY_SUBMITTED_PATTERNS` from constants, consistent with
/// `is_already_submitted_error` in `domain::transaction::evm::evm_transaction`.
fn is_non_retriable_transaction_rpc_message(message: &str) -> bool {
    let msg_lower = message.to_lowercase();
    ALREADY_SUBMITTED_PATTERNS
        .iter()
        .any(|p| msg_lower.contains(p))
        || NONCE_TOO_HIGH_PATTERNS
            .iter()
            .any(|p| msg_lower.contains(p))
        || matches_known_transaction(&msg_lower)
}

// Errors that are retriable
pub fn is_retriable_error(error: &ProviderError) -> bool {
    match error {
        // HTTP-level errors that are retriable
        ProviderError::Timeout
        | ProviderError::RateLimited
        | ProviderError::BadGateway
        | ProviderError::TransportError(_) => true,

        ProviderError::RequestError { status_code, .. } => {
            match *status_code {
                // Non-retriable 5xx: persistent server-side issues
                501 | 505 => false, // Not Implemented, HTTP Version Not Supported

                // Retriable 5xx: temporary server-side issues
                500 | 502..=504 | 506..=599 => true,

                // Retriable 4xx: timeout or rate-limit related
                408 | 425 | 429 => true,

                // Non-retriable 4xx: client errors
                400..=499 => false,

                // Other status codes: not retriable
                _ => false,
            }
        }

        // JSON-RPC error codes (EIP-1474)
        ProviderError::RpcErrorCode { code, message } => {
            match code {
                // -32002: Resource unavailable — retriable unless the message indicates a
                // transaction-level rejection (some providers wrap nonce/tx errors here)
                -32002 => !is_non_retriable_transaction_rpc_message(message),
                // -32005: Limit exceeded / rate limited
                -32005 => true,
                // -32603: Internal error — retriable unless the message indicates a
                // transaction-level rejection (some providers wrap nonce/tx errors here)
                -32603 => !is_non_retriable_transaction_rpc_message(message),
                // -32000: Invalid input
                -32000 => false,
                // -32001: Resource not found
                -32001 => false,
                // -32003: Transaction rejected
                -32003 => false,
                // -32004: Method not supported
                -32004 => false,

                // Standard JSON-RPC 2.0 errors (not retriable)
                // -32700: Parse error
                // -32600: Invalid request
                // -32601: Method not found
                // -32602: Invalid params
                -32700..=-32600 => false,

                // All other error codes: not retriable by default
                _ => false,
            }
        }

        ProviderError::SolanaRpcError(err) => err.is_transient(),

        // Any other errors: check message for network-related issues
        _ => {
            let err_msg = format!("{error}");
            let msg_lower = err_msg.to_lowercase();
            msg_lower.contains("timeout")
                || msg_lower.contains("connection")
                || msg_lower.contains("reset")
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use lazy_static::lazy_static;
    use std::env;
    use std::sync::Mutex;
    use std::time::Duration;

    // Use a mutex to ensure tests don't run in parallel when modifying env vars
    lazy_static! {
        static ref ENV_MUTEX: Mutex<()> = Mutex::new(());
    }

    fn setup_test_env() {
        env::set_var("API_KEY", "7EF1CB7C-5003-4696-B384-C72AF8C3E15D"); // noboost
        env::set_var("REDIS_URL", "redis://localhost:6379");
        env::set_var("RPC_TIMEOUT_MS", "5000");
    }

    fn cleanup_test_env() {
        env::remove_var("API_KEY");
        env::remove_var("REDIS_URL");
        env::remove_var("RPC_TIMEOUT_MS");
    }

    fn create_test_evm_network() -> EvmNetwork {
        EvmNetwork {
            network: "test-evm".to_string(),
            rpc_urls: vec![RpcConfig::new("https://rpc.example.com".to_string())],
            explorer_urls: None,
            average_blocktime_ms: 12000,
            is_testnet: true,
            tags: vec![],
            chain_id: 1337,
            required_confirmations: 1,
            features: vec![],
            symbol: "ETH".to_string(),
            gas_price_cache: None,
        }
    }

    fn create_test_solana_network(network_str: &str) -> SolanaNetwork {
        SolanaNetwork {
            network: network_str.to_string(),
            rpc_urls: vec![RpcConfig::new("https://api.testnet.solana.com".to_string())],
            explorer_urls: None,
            average_blocktime_ms: 400,
            is_testnet: true,
            tags: vec![],
        }
    }

    fn create_test_stellar_network() -> StellarNetwork {
        StellarNetwork {
            network: "testnet".to_string(),
            rpc_urls: vec![RpcConfig::new(
                "https://soroban-testnet.stellar.org".to_string(),
            )],
            explorer_urls: None,
            average_blocktime_ms: 5000,
            is_testnet: true,
            tags: vec![],
            passphrase: "Test SDF Network ; September 2015".to_string(),
            horizon_url: Some("https://horizon-testnet.stellar.org".to_string()),
        }
    }

    #[test]
    fn test_from_hex_error() {
        let hex_error = hex::FromHexError::OddLength;
        let provider_error: ProviderError = hex_error.into();
        assert!(matches!(provider_error, ProviderError::InvalidAddress(_)));
    }

    #[test]
    fn test_from_addr_parse_error() {
        let addr_error = "invalid:address"
            .parse::<std::net::SocketAddr>()
            .unwrap_err();
        let provider_error: ProviderError = addr_error.into();
        assert!(matches!(
            provider_error,
            ProviderError::NetworkConfiguration(_)
        ));
    }

    #[test]
    fn test_from_parse_int_error() {
        let parse_error = "not_a_number".parse::<u64>().unwrap_err();
        let provider_error: ProviderError = parse_error.into();
        assert!(matches!(provider_error, ProviderError::Other(_)));
    }

    #[actix_rt::test]
    async fn test_categorize_reqwest_error_timeout() {
        let client = reqwest::Client::new();
        let timeout_err = client
            .get("http://example.com")
            .timeout(Duration::from_nanos(1))
            .send()
            .await
            .unwrap_err();

        assert!(timeout_err.is_timeout());

        let provider_error = categorize_reqwest_error(&timeout_err);
        assert!(matches!(provider_error, ProviderError::Timeout));
    }

    #[actix_rt::test]
    async fn test_categorize_reqwest_error_rate_limited() {
        let mut mock_server = mockito::Server::new_async().await;

        let _mock = mock_server
            .mock("GET", mockito::Matcher::Any)
            .with_status(429)
            .create_async()
            .await;

        let client = reqwest::Client::new();
        let response = client
            .get(mock_server.url())
            .send()
            .await
            .expect("Failed to get response");

        let err = response
            .error_for_status()
            .expect_err("Expected error for status 429");

        assert!(err.status().is_some());
        assert_eq!(err.status().unwrap().as_u16(), 429);

        let provider_error = categorize_reqwest_error(&err);
        assert!(matches!(provider_error, ProviderError::RateLimited));
    }

    #[actix_rt::test]
    async fn test_categorize_reqwest_error_bad_gateway() {
        let mut mock_server = mockito::Server::new_async().await;

        let _mock = mock_server
            .mock("GET", mockito::Matcher::Any)
            .with_status(502)
            .create_async()
            .await;

        let client = reqwest::Client::new();
        let response = client
            .get(mock_server.url())
            .send()
            .await
            .expect("Failed to get response");

        let err = response
            .error_for_status()
            .expect_err("Expected error for status 502");

        assert!(err.status().is_some());
        assert_eq!(err.status().unwrap().as_u16(), 502);

        let provider_error = categorize_reqwest_error(&err);
        assert!(matches!(provider_error, ProviderError::BadGateway));
    }

    #[actix_rt::test]
    async fn test_categorize_reqwest_error_other() {
        let client = reqwest::Client::new();
        let err = client
            .get("http://non-existent-host-12345.local")
            .send()
            .await
            .unwrap_err();

        assert!(!err.is_timeout());
        assert!(err.status().is_none()); // No status code

        let provider_error = categorize_reqwest_error(&err);
        assert!(matches!(provider_error, ProviderError::Other(_)));
    }

    #[test]
    fn test_from_eyre_report_other_error() {
        let eyre_error: eyre::Report = eyre::eyre!("Generic error");
        let provider_error: ProviderError = eyre_error.into();
        assert!(matches!(provider_error, ProviderError::Other(_)));
    }

    #[test]
    fn test_get_evm_network_provider_valid_network() {
        let _lock = ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner());
        setup_test_env();

        let network = create_test_evm_network();
        let result = get_network_provider(&network, None);

        cleanup_test_env();
        assert!(result.is_ok());
    }

    #[test]
    fn test_get_evm_network_provider_with_custom_urls() {
        let _lock = ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner());
        setup_test_env();

        let network = create_test_evm_network();
        let custom_urls = vec![
            RpcConfig {
                url: "https://custom-rpc1.example.com".to_string(),
                weight: 1,
                ..Default::default()
            },
            RpcConfig {
                url: "https://custom-rpc2.example.com".to_string(),
                weight: 1,
                ..Default::default()
            },
        ];
        let result = get_network_provider(&network, Some(custom_urls));

        cleanup_test_env();
        assert!(result.is_ok());
    }

    #[test]
    fn test_get_evm_network_provider_with_empty_custom_urls() {
        let _lock = ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner());
        setup_test_env();

        let network = create_test_evm_network();
        let custom_urls: Vec<RpcConfig> = vec![];
        let result = get_network_provider(&network, Some(custom_urls));

        cleanup_test_env();
        assert!(result.is_ok()); // Should fall back to public URLs
    }

    #[test]
    fn test_get_solana_network_provider_valid_network_mainnet_beta() {
        let _lock = ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner());
        setup_test_env();

        let network = create_test_solana_network("mainnet-beta");
        let result = get_network_provider(&network, None);

        cleanup_test_env();
        assert!(result.is_ok());
    }

    #[test]
    fn test_get_solana_network_provider_valid_network_testnet() {
        let _lock = ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner());
        setup_test_env();

        let network = create_test_solana_network("testnet");
        let result = get_network_provider(&network, None);

        cleanup_test_env();
        assert!(result.is_ok());
    }

    #[test]
    fn test_get_solana_network_provider_with_custom_urls() {
        let _lock = ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner());
        setup_test_env();

        let network = create_test_solana_network("testnet");
        let custom_urls = vec![
            RpcConfig {
                url: "https://custom-rpc1.example.com".to_string(),
                weight: 1,
                ..Default::default()
            },
            RpcConfig {
                url: "https://custom-rpc2.example.com".to_string(),
                weight: 1,
                ..Default::default()
            },
        ];
        let result = get_network_provider(&network, Some(custom_urls));

        cleanup_test_env();
        assert!(result.is_ok());
    }

    #[test]
    fn test_get_solana_network_provider_with_empty_custom_urls() {
        let _lock = ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner());
        setup_test_env();

        let network = create_test_solana_network("testnet");
        let custom_urls: Vec<RpcConfig> = vec![];
        let result = get_network_provider(&network, Some(custom_urls));

        cleanup_test_env();
        assert!(result.is_ok()); // Should fall back to public URLs
    }

    // Tests for Stellar Network Provider
    #[test]
    fn test_get_stellar_network_provider_valid_network_fallback_public() {
        let _lock = ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner());
        setup_test_env();

        let network = create_test_stellar_network();
        let result = get_network_provider(&network, None); // No custom URLs

        cleanup_test_env();
        assert!(result.is_ok()); // Should fall back to public URLs for testnet
                                 // StellarProvider::new will use the first public URL: https://soroban-testnet.stellar.org
    }

    #[test]
    fn test_get_stellar_network_provider_with_custom_urls() {
        let _lock = ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner());
        setup_test_env();

        let network = create_test_stellar_network();
        let custom_urls = vec![
            RpcConfig::new("https://custom-stellar-rpc1.example.com".to_string()),
            RpcConfig::with_weight("http://custom-stellar-rpc2.example.com".to_string(), 50)
                .unwrap(),
        ];
        let result = get_network_provider(&network, Some(custom_urls));

        cleanup_test_env();
        assert!(result.is_ok());
        // StellarProvider::new will pick custom-stellar-rpc1 (default weight 100) over custom-stellar-rpc2 (weight 50)
    }

    #[test]
    fn test_get_stellar_network_provider_with_empty_custom_urls_fallback() {
        let _lock = ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner());
        setup_test_env();

        let network = create_test_stellar_network();
        let custom_urls: Vec<RpcConfig> = vec![]; // Empty custom URLs
        let result = get_network_provider(&network, Some(custom_urls));

        cleanup_test_env();
        assert!(result.is_ok()); // Should fall back to public URLs for mainnet
                                 // StellarProvider::new will use the first public URL: https://horizon.stellar.org
    }

    #[test]
    fn test_get_stellar_network_provider_custom_urls_with_zero_weight() {
        let _lock = ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner());
        setup_test_env();

        let network = create_test_stellar_network();
        let custom_urls = vec![
            RpcConfig::with_weight("http://zero-weight-rpc.example.com".to_string(), 0).unwrap(),
            RpcConfig::new("http://active-rpc.example.com".to_string()), // Default weight 100
        ];
        let result = get_network_provider(&network, Some(custom_urls));
        cleanup_test_env();
        assert!(result.is_ok()); // active-rpc should be chosen
    }

    #[test]
    fn test_get_stellar_network_provider_all_custom_urls_zero_weight_fallback() {
        let _lock = ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner());
        setup_test_env();

        let network = create_test_stellar_network();
        let custom_urls = vec![
            RpcConfig::with_weight("http://zero1.example.com".to_string(), 0).unwrap(),
            RpcConfig::with_weight("http://zero2.example.com".to_string(), 0).unwrap(),
        ];
        // Since StellarProvider::new filters out zero-weight URLs, and if the list becomes empty,
        // get_network_provider does NOT re-trigger fallback to public. Instead, StellarProvider::new itself will error.
        // The current get_network_provider logic passes the custom_urls to N::new_provider if Some and not empty.
        // If custom_urls becomes effectively empty *inside* N::new_provider (like StellarProvider::new after filtering weights),
        // then N::new_provider is responsible for erroring or handling.
        let result = get_network_provider(&network, Some(custom_urls));
        cleanup_test_env();
        assert!(result.is_err());
        match result.unwrap_err() {
            ProviderError::NetworkConfiguration(msg) => {
                assert!(msg.contains("No active RPC configurations provided"));
            }
            _ => panic!("Unexpected error type"),
        }
    }

    #[test]
    fn test_provider_error_rpc_error_code_variant() {
        let error = ProviderError::RpcErrorCode {
            code: -32000,
            message: "insufficient funds".to_string(),
        };
        let error_string = format!("{error}");
        assert!(error_string.contains("-32000"));
        assert!(error_string.contains("insufficient funds"));
    }

    #[test]
    fn test_get_stellar_network_provider_invalid_custom_url_scheme() {
        let _lock = ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner());
        setup_test_env();
        let network = create_test_stellar_network();
        let custom_urls = vec![RpcConfig::new("ftp://custom-ftp.example.com".to_string())];
        let result = get_network_provider(&network, Some(custom_urls));
        cleanup_test_env();
        assert!(result.is_err());
        match result.unwrap_err() {
            ProviderError::NetworkConfiguration(msg) => {
                // This error comes from RpcConfig::validate_list inside StellarProvider::new
                assert!(msg.contains("Invalid URL scheme"));
            }
            _ => panic!("Unexpected error type"),
        }
    }

    #[test]
    fn test_should_mark_provider_failed_server_errors() {
        // 5xx errors should mark provider as failed
        for status_code in 500..=599 {
            let error = ProviderError::RequestError {
                error: format!("Server error {status_code}"),
                status_code,
            };
            assert!(
                should_mark_provider_failed(&error),
                "Status code {status_code} should mark provider as failed"
            );
        }
    }

    #[test]
    fn test_should_mark_provider_failed_auth_errors() {
        // Authentication/authorization errors should mark provider as failed
        let auth_errors = [401, 403];
        for &status_code in &auth_errors {
            let error = ProviderError::RequestError {
                error: format!("Auth error {status_code}"),
                status_code,
            };
            assert!(
                should_mark_provider_failed(&error),
                "Status code {status_code} should mark provider as failed"
            );
        }
    }

    #[test]
    fn test_should_mark_provider_failed_not_found_errors() {
        // 404 and 410 should mark provider as failed (endpoint issues)
        let not_found_errors = [404, 410];
        for &status_code in &not_found_errors {
            let error = ProviderError::RequestError {
                error: format!("Not found error {status_code}"),
                status_code,
            };
            assert!(
                should_mark_provider_failed(&error),
                "Status code {status_code} should mark provider as failed"
            );
        }
    }

    #[test]
    fn test_should_mark_provider_failed_client_errors_not_failed() {
        // These 4xx errors should NOT mark provider as failed (client-side issues)
        let client_errors = [400, 405, 413, 414, 415, 422, 429];
        for &status_code in &client_errors {
            let error = ProviderError::RequestError {
                error: format!("Client error {status_code}"),
                status_code,
            };
            assert!(
                !should_mark_provider_failed(&error),
                "Status code {status_code} should NOT mark provider as failed"
            );
        }
    }

    #[test]
    fn test_should_mark_provider_failed_other_error_types() {
        // Test non-RequestError types - these should NOT mark provider as failed
        let errors = [
            ProviderError::Timeout,
            ProviderError::RateLimited,
            ProviderError::BadGateway,
            ProviderError::InvalidAddress("test".to_string()),
            ProviderError::NetworkConfiguration("test".to_string()),
            ProviderError::Other("test".to_string()),
        ];

        for error in errors {
            assert!(
                !should_mark_provider_failed(&error),
                "Error type {error:?} should NOT mark provider as failed"
            );
        }
    }

    #[test]
    fn test_should_mark_provider_failed_edge_cases() {
        // Test some edge case status codes
        let edge_cases = [
            (200, false), // Success - shouldn't happen in error context but test anyway
            (300, false), // Redirection
            (418, false), // I'm a teapot - should not mark as failed
            (451, false), // Unavailable for legal reasons - client issue
            (499, false), // Client closed request - client issue
        ];

        for (status_code, should_fail) in edge_cases {
            let error = ProviderError::RequestError {
                error: format!("Edge case error {status_code}"),
                status_code,
            };
            assert_eq!(
                should_mark_provider_failed(&error),
                should_fail,
                "Status code {} should {} mark provider as failed",
                status_code,
                if should_fail { "" } else { "NOT" }
            );
        }
    }

    #[test]
    fn test_is_retriable_error_retriable_types() {
        // These error types should be retriable
        let retriable_errors = [
            ProviderError::Timeout,
            ProviderError::RateLimited,
            ProviderError::BadGateway,
            ProviderError::TransportError("test".to_string()),
        ];

        for error in retriable_errors {
            assert!(
                is_retriable_error(&error),
                "Error type {error:?} should be retriable"
            );
        }
    }

    #[test]
    fn test_is_retriable_error_non_retriable_types() {
        // These error types should NOT be retriable
        let non_retriable_errors = [
            ProviderError::InvalidAddress("test".to_string()),
            ProviderError::NetworkConfiguration("test".to_string()),
            ProviderError::RequestError {
                error: "Some error".to_string(),
                status_code: 400,
            },
        ];

        for error in non_retriable_errors {
            assert!(
                !is_retriable_error(&error),
                "Error type {error:?} should NOT be retriable"
            );
        }
    }

    #[test]
    fn test_is_retriable_error_message_based_detection() {
        // Test errors that should be retriable based on message content
        let retriable_messages = [
            "Connection timeout occurred",
            "Network connection reset",
            "Connection refused",
            "TIMEOUT error happened",
            "Connection was reset by peer",
        ];

        for message in retriable_messages {
            let error = ProviderError::Other(message.to_string());
            assert!(
                is_retriable_error(&error),
                "Error with message '{message}' should be retriable"
            );
        }
    }

    #[test]
    fn test_is_retriable_error_message_based_non_retriable() {
        // Test errors that should NOT be retriable based on message content
        let non_retriable_messages = [
            "Invalid address format",
            "Bad request parameters",
            "Authentication failed",
            "Method not found",
            "Some other error",
        ];

        for message in non_retriable_messages {
            let error = ProviderError::Other(message.to_string());
            assert!(
                !is_retriable_error(&error),
                "Error with message '{message}' should NOT be retriable"
            );
        }
    }

    #[test]
    fn test_is_retriable_error_case_insensitive() {
        // Test that message-based detection is case insensitive
        let case_variations = [
            "TIMEOUT",
            "Timeout",
            "timeout",
            "CONNECTION",
            "Connection",
            "connection",
            "RESET",
            "Reset",
            "reset",
        ];

        for message in case_variations {
            let error = ProviderError::Other(message.to_string());
            assert!(
                is_retriable_error(&error),
                "Error with message '{message}' should be retriable (case insensitive)"
            );
        }
    }

    #[test]
    fn test_is_retriable_error_request_error_retriable_5xx() {
        // Test retriable 5xx status codes
        let retriable_5xx = vec![
            (500, "Internal Server Error"),
            (502, "Bad Gateway"),
            (503, "Service Unavailable"),
            (504, "Gateway Timeout"),
            (506, "Variant Also Negotiates"),
            (507, "Insufficient Storage"),
            (508, "Loop Detected"),
            (510, "Not Extended"),
            (511, "Network Authentication Required"),
            (599, "Network Connect Timeout Error"),
        ];

        for (status_code, description) in retriable_5xx {
            let error = ProviderError::RequestError {
                error: description.to_string(),
                status_code,
            };
            assert!(
                is_retriable_error(&error),
                "Status code {status_code} ({description}) should be retriable"
            );
        }
    }

    #[test]
    fn test_is_retriable_error_request_error_non_retriable_5xx() {
        // Test non-retriable 5xx status codes (persistent server issues)
        let non_retriable_5xx = vec![
            (501, "Not Implemented"),
            (505, "HTTP Version Not Supported"),
        ];

        for (status_code, description) in non_retriable_5xx {
            let error = ProviderError::RequestError {
                error: description.to_string(),
                status_code,
            };
            assert!(
                !is_retriable_error(&error),
                "Status code {status_code} ({description}) should NOT be retriable"
            );
        }
    }

    #[test]
    fn test_is_retriable_error_request_error_retriable_4xx() {
        // Test retriable 4xx status codes (timeout/rate-limit related)
        let retriable_4xx = vec![
            (408, "Request Timeout"),
            (425, "Too Early"),
            (429, "Too Many Requests"),
        ];

        for (status_code, description) in retriable_4xx {
            let error = ProviderError::RequestError {
                error: description.to_string(),
                status_code,
            };
            assert!(
                is_retriable_error(&error),
                "Status code {status_code} ({description}) should be retriable"
            );
        }
    }

    #[test]
    fn test_is_retriable_error_request_error_non_retriable_4xx() {
        // Test non-retriable 4xx status codes (client errors)
        let non_retriable_4xx = vec![
            (400, "Bad Request"),
            (401, "Unauthorized"),
            (403, "Forbidden"),
            (404, "Not Found"),
            (405, "Method Not Allowed"),
            (406, "Not Acceptable"),
            (407, "Proxy Authentication Required"),
            (409, "Conflict"),
            (410, "Gone"),
            (411, "Length Required"),
            (412, "Precondition Failed"),
            (413, "Payload Too Large"),
            (414, "URI Too Long"),
            (415, "Unsupported Media Type"),
            (416, "Range Not Satisfiable"),
            (417, "Expectation Failed"),
            (418, "I'm a teapot"),
            (421, "Misdirected Request"),
            (422, "Unprocessable Entity"),
            (423, "Locked"),
            (424, "Failed Dependency"),
            (426, "Upgrade Required"),
            (428, "Precondition Required"),
            (431, "Request Header Fields Too Large"),
            (451, "Unavailable For Legal Reasons"),
            (499, "Client Closed Request"),
        ];

        for (status_code, description) in non_retriable_4xx {
            let error = ProviderError::RequestError {
                error: description.to_string(),
                status_code,
            };
            assert!(
                !is_retriable_error(&error),
                "Status code {status_code} ({description}) should NOT be retriable"
            );
        }
    }

    #[test]
    fn test_is_retriable_error_request_error_other_status_codes() {
        // Test other status codes (1xx, 2xx, 3xx) - should not be retriable
        let other_status_codes = vec![
            (100, "Continue"),
            (101, "Switching Protocols"),
            (200, "OK"),
            (201, "Created"),
            (204, "No Content"),
            (300, "Multiple Choices"),
            (301, "Moved Permanently"),
            (302, "Found"),
            (304, "Not Modified"),
            (600, "Custom status"),
            (999, "Unknown status"),
        ];

        for (status_code, description) in other_status_codes {
            let error = ProviderError::RequestError {
                error: description.to_string(),
                status_code,
            };
            assert!(
                !is_retriable_error(&error),
                "Status code {status_code} ({description}) should NOT be retriable"
            );
        }
    }

    #[test]
    fn test_is_retriable_error_request_error_boundary_cases() {
        // Test boundary cases for our ranges
        let test_cases = vec![
            // Just before retriable 4xx range
            (407, false, "Proxy Authentication Required"),
            (408, true, "Request Timeout - first retriable 4xx"),
            (409, false, "Conflict"),
            // Around 425
            (424, false, "Failed Dependency"),
            (425, true, "Too Early"),
            (426, false, "Upgrade Required"),
            // Around 429
            (428, false, "Precondition Required"),
            (429, true, "Too Many Requests"),
            (430, false, "Would be non-retriable if it existed"),
            // 5xx boundaries
            (499, false, "Last 4xx"),
            (500, true, "First 5xx - retriable"),
            (501, false, "Not Implemented - exception"),
            (502, true, "Bad Gateway - retriable"),
            (505, false, "HTTP Version Not Supported - exception"),
            (506, true, "First after 505 exception"),
            (599, true, "Last defined 5xx"),
        ];

        for (status_code, should_be_retriable, description) in test_cases {
            let error = ProviderError::RequestError {
                error: description.to_string(),
                status_code,
            };
            assert_eq!(
                is_retriable_error(&error),
                should_be_retriable,
                "Status code {} ({}) should{} be retriable",
                status_code,
                description,
                if should_be_retriable { "" } else { " NOT" }
            );
        }
    }

    #[test]
    fn test_is_non_retriable_transaction_rpc_message() {
        // Positive cases: these messages should be recognized as non-retriable
        assert!(is_non_retriable_transaction_rpc_message("nonce too low"));
        assert!(is_non_retriable_transaction_rpc_message("Nonce Too Low"));
        assert!(is_non_retriable_transaction_rpc_message("nonce is too low"));
        assert!(is_non_retriable_transaction_rpc_message("already known"));
        assert!(is_non_retriable_transaction_rpc_message(
            "known transaction"
        ));
        assert!(is_non_retriable_transaction_rpc_message(
            "Known Transaction"
        ));
        assert!(is_non_retriable_transaction_rpc_message(
            "replacement transaction underpriced"
        ));
        assert!(is_non_retriable_transaction_rpc_message(
            "same hash was already imported"
        ));
        assert!(is_non_retriable_transaction_rpc_message(
            "Transaction nonce too low"
        ));

        // Negative cases: generic/unrelated messages should not match
        assert!(!is_non_retriable_transaction_rpc_message("Internal error"));
        assert!(!is_non_retriable_transaction_rpc_message("server busy"));
        assert!(!is_non_retriable_transaction_rpc_message(""));
        // "unknown transaction" must NOT match "known transaction"
        assert!(!is_non_retriable_transaction_rpc_message(
            "Unknown transaction status"
        ));

        // Nonce-too-high patterns are also non-retriable
        assert!(is_non_retriable_transaction_rpc_message("nonce too high"));
        assert!(is_non_retriable_transaction_rpc_message(
            "nonce too far in the future",
        ));
        assert!(is_non_retriable_transaction_rpc_message(
            "exceeds next nonce"
        ));
        assert!(is_non_retriable_transaction_rpc_message(
            "Nonce Too Far In The Future"
        ));
    }

    #[test]
    fn test_is_retriable_error_rpc_tx_errors_not_retriable() {
        // Transaction-level messages that should NOT be retriable regardless of code
        let non_retriable_messages = vec![
            "Transaction nonce too low",
            "nonce too low",
            "nonce is too low",
            "already known",
            "known transaction",
            "replacement transaction underpriced",
            "same hash was already imported",
        ];

        // Messages that should remain retriable (generic/unrelated)
        let retriable_messages = vec![
            "Internal error",
            "",
            // "unknown transaction" must NOT false-positive on "known transaction"
            "Unknown transaction status",
            "Resource unavailable",
        ];

        // Both -32603 and -32002 should behave the same way for tx-level messages
        for code in [-32603, -32002] {
            for message in &non_retriable_messages {
                let error = ProviderError::RpcErrorCode {
                    code,
                    message: message.to_string(),
                };
                assert!(
                    !is_retriable_error(&error),
                    "{code} with message {message:?} should NOT be retriable"
                );
            }

            for message in &retriable_messages {
                let error = ProviderError::RpcErrorCode {
                    code,
                    message: message.to_string(),
                };
                assert!(
                    is_retriable_error(&error),
                    "{code} with message {message:?} should be retriable"
                );
            }
        }
    }
}