1use alloy::network::ReceiptResponse;
6use chrono::{DateTime, Duration, Utc};
7use eyre::Result;
8use tracing::{debug, error, warn};
9
10use super::super::common::is_active_nonce_status;
11use super::EvmRelayerTransaction;
12use super::{
13 ensure_status, evm_transaction::TX_NONCE_RECONCILE_TRIGGER, get_age_since_status_change,
14 has_enough_confirmations, is_noop, is_too_early_to_resubmit, is_transaction_valid, make_noop,
15 too_many_attempts, too_many_noop_attempts,
16};
17use crate::constants::{
18 get_evm_min_age_for_hash_recovery, get_evm_pending_recovery_trigger_timeout,
19 get_evm_prepare_timeout, get_evm_resend_timeout, ARBITRUM_TIME_TO_RESUBMIT,
20 EVM_MIN_HASHES_FOR_RECOVERY, MAX_GAP_SCAN_RANGE,
21};
22use crate::domain::transaction::common::{
23 get_age_of_sent_at, is_final_state, is_pending_transaction,
24};
25use crate::domain::transaction::util::get_age_since_created;
26use crate::models::{EvmNetwork, NetworkRepoModel, NetworkType};
27use crate::repositories::{NetworkRepository, RelayerRepository};
28use crate::{
29 domain::transaction::evm::price_calculator::PriceCalculatorTrait,
30 jobs::{JobProducerTrait, StatusCheckContext},
31 models::{
32 NetworkTransactionData, RelayerRepoModel, TransactionError, TransactionRepoModel,
33 TransactionStatus, TransactionUpdateRequest,
34 },
35 repositories::{Repository, TransactionCounterTrait, TransactionRepository},
36 services::{provider::EvmProviderTrait, signer::Signer},
37 utils::{get_resubmit_timeout_for_speed, get_resubmit_timeout_with_backoff},
38};
39
40impl<P, RR, NR, TR, J, S, TCR, PC> EvmRelayerTransaction<P, RR, NR, TR, J, S, TCR, PC>
41where
42 P: EvmProviderTrait + Send + Sync,
43 RR: RelayerRepository + Repository<RelayerRepoModel, String> + Send + Sync + 'static,
44 NR: NetworkRepository + Repository<NetworkRepoModel, String> + Send + Sync + 'static,
45 TR: TransactionRepository + Repository<TransactionRepoModel, String> + Send + Sync + 'static,
46 J: JobProducerTrait + Send + Sync + 'static,
47 S: Signer + Send + Sync + 'static,
48 TCR: TransactionCounterTrait + Send + Sync + 'static,
49 PC: PriceCalculatorTrait + Send + Sync,
50{
51 pub(super) async fn check_transaction_status(
52 &self,
53 tx: &TransactionRepoModel,
54 ) -> Result<TransactionStatus, TransactionError> {
55 if is_final_state(&tx.status) {
57 return Ok(tx.status.clone());
58 }
59
60 match tx.status {
63 TransactionStatus::Pending | TransactionStatus::Sent => {
64 return Ok(tx.status.clone());
65 }
66 _ => {}
67 }
68
69 let evm_data = tx.network_data.get_evm_transaction_data()?;
70 let tx_hash = evm_data
71 .hash
72 .as_ref()
73 .ok_or(TransactionError::UnexpectedError(
74 "Transaction hash is missing".to_string(),
75 ))?;
76
77 let receipt_result = self.provider().get_transaction_receipt(tx_hash).await?;
78
79 if let Some(receipt) = receipt_result {
80 if !receipt.inner.status() {
81 return Ok(TransactionStatus::Failed);
82 }
83 let last_block_number = self.provider().get_block_number().await?;
84 let tx_block_number = receipt
85 .block_number
86 .ok_or(TransactionError::UnexpectedError(
87 "Transaction receipt missing block number".to_string(),
88 ))?;
89
90 let network_model = self
91 .network_repository()
92 .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
93 .await?
94 .ok_or(TransactionError::UnexpectedError(format!(
95 "Network with chain id {} not found",
96 evm_data.chain_id
97 )))?;
98
99 let network = EvmNetwork::try_from(network_model).map_err(|e| {
100 TransactionError::UnexpectedError(format!(
101 "Error converting network model to EvmNetwork: {e}"
102 ))
103 })?;
104
105 if !has_enough_confirmations(
106 tx_block_number,
107 last_block_number,
108 network.required_confirmations,
109 ) {
110 debug!(
111 tx_id = %tx.id,
112 relayer_id = %tx.relayer_id,
113 tx_hash = %tx_hash,
114 "transaction mined but not confirmed"
115 );
116 return Ok(TransactionStatus::Mined);
117 }
118 Ok(TransactionStatus::Confirmed)
119 } else {
120 debug!(
121 tx_id = %tx.id,
122 relayer_id = %tx.relayer_id,
123 tx_hash = %tx_hash,
124 "transaction not yet mined"
125 );
126
127 if tx.hashes.len() > 1 && self.should_try_hash_recovery(tx)? {
131 if let Some(recovered_tx) = self
132 .try_recover_with_historical_hashes(tx, &evm_data)
133 .await?
134 {
135 return Ok(recovered_tx.status);
137 }
138 }
139
140 Ok(TransactionStatus::Submitted)
141 }
142 }
143
144 pub(super) async fn should_resubmit(
146 &self,
147 tx: &TransactionRepoModel,
148 ) -> Result<bool, TransactionError> {
149 ensure_status(tx, TransactionStatus::Submitted, Some("should_resubmit"))?;
151
152 let evm_data = tx.network_data.get_evm_transaction_data()?;
153 let age = get_age_of_sent_at(tx)?;
154
155 let network_model = self
157 .network_repository()
158 .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
159 .await?
160 .ok_or(TransactionError::UnexpectedError(format!(
161 "Network with chain id {} not found",
162 evm_data.chain_id
163 )))?;
164
165 let network = EvmNetwork::try_from(network_model).map_err(|e| {
166 TransactionError::UnexpectedError(format!(
167 "Error converting network model to EvmNetwork: {e}"
168 ))
169 })?;
170
171 let timeout = match network.is_arbitrum() {
172 true => ARBITRUM_TIME_TO_RESUBMIT,
173 false => get_resubmit_timeout_for_speed(&evm_data.speed),
174 };
175
176 let timeout_with_backoff = match network.is_arbitrum() {
177 true => timeout, false => get_resubmit_timeout_with_backoff(timeout, tx.hashes.len()),
179 };
180
181 if age > Duration::milliseconds(timeout_with_backoff) {
182 debug!(
183 tx_id = %tx.id,
184 relayer_id = %tx.relayer_id,
185 age_ms = %age.num_milliseconds(),
186 "transaction has been pending for too long, resubmitting"
187 );
188 return Ok(true);
189 }
190 Ok(false)
191 }
192
193 pub(super) async fn should_noop(
203 &self,
204 tx: &TransactionRepoModel,
205 ) -> Result<(bool, Option<String>), TransactionError> {
206 if too_many_noop_attempts(tx) {
207 debug!("Transaction has too many NOOP attempts already");
208 return Ok((false, None));
209 }
210
211 let evm_data = tx.network_data.get_evm_transaction_data()?;
212 if is_noop(&evm_data) {
213 return Ok((false, None));
214 }
215
216 let network_model = self
217 .network_repository()
218 .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
219 .await?
220 .ok_or(TransactionError::UnexpectedError(format!(
221 "Network with chain id {} not found",
222 evm_data.chain_id
223 )))?;
224
225 let network = EvmNetwork::try_from(network_model).map_err(|e| {
226 TransactionError::UnexpectedError(format!(
227 "Error converting network model to EvmNetwork: {e}"
228 ))
229 })?;
230
231 if network.is_rollup() && too_many_attempts(tx) {
232 let reason =
233 "Rollup transaction has too many attempts. Replacing with NOOP.".to_string();
234 debug!(
235 tx_id = %tx.id,
236 relayer_id = %tx.relayer_id,
237 reason = %reason,
238 "replacing transaction with NOOP"
239 );
240 return Ok((true, Some(reason)));
241 }
242
243 if !is_transaction_valid(&tx.created_at, &tx.valid_until) {
244 let reason = "Transaction is expired. Replacing with NOOP.".to_string();
245 debug!(
246 tx_id = %tx.id,
247 relayer_id = %tx.relayer_id,
248 reason = %reason,
249 "replacing transaction with NOOP"
250 );
251 return Ok((true, Some(reason)));
252 }
253
254 if tx.status == TransactionStatus::Pending {
255 let created_at = &tx.created_at;
256 let created_time = DateTime::parse_from_rfc3339(created_at)
257 .map_err(|e| {
258 TransactionError::UnexpectedError(format!("Invalid created_at timestamp: {e}"))
259 })?
260 .with_timezone(&Utc);
261 let age = Utc::now().signed_duration_since(created_time);
262 if age > get_evm_prepare_timeout() {
263 let reason = format!(
264 "Transaction in Pending state for over {} minutes. Replacing with NOOP.",
265 get_evm_prepare_timeout().num_minutes()
266 );
267 debug!(
268 tx_id = %tx.id,
269 relayer_id = %tx.relayer_id,
270 reason = %reason,
271 "replacing transaction with NOOP"
272 );
273 return Ok((true, Some(reason)));
274 }
275 }
276
277 let latest_block = self.provider().get_block_by_number().await;
278 if let Ok(block) = latest_block {
279 let block_gas_limit = block.header.gas_limit;
280 if let Some(gas_limit) = evm_data.gas_limit {
281 if gas_limit > block_gas_limit {
282 let reason = format!(
283 "Transaction gas limit ({gas_limit}) exceeds block gas limit ({block_gas_limit}). Replacing with NOOP.",
284 );
285 warn!(
286 tx_id = %tx.id,
287 tx_gas_limit = %gas_limit,
288 block_gas_limit = %block_gas_limit,
289 "transaction gas limit exceeds block gas limit, replacing with NOOP"
290 );
291 return Ok((true, Some(reason)));
292 }
293 }
294 }
295
296 Ok((false, None))
297 }
298
299 pub(super) async fn update_transaction_status_if_needed(
301 &self,
302 tx: TransactionRepoModel,
303 new_status: TransactionStatus,
304 status_reason: Option<String>,
305 ) -> Result<TransactionRepoModel, TransactionError> {
306 if tx.status != new_status {
307 return self
308 .update_transaction_status(tx, new_status, status_reason)
309 .await;
310 }
311 Ok(tx)
312 }
313
314 pub(super) async fn prepare_noop_update_request(
316 &self,
317 tx: &TransactionRepoModel,
318 is_cancellation: bool,
319 reason: Option<String>,
320 ) -> Result<TransactionUpdateRequest, TransactionError> {
321 let mut evm_data = tx.network_data.get_evm_transaction_data()?;
322 let network_model = self
323 .network_repository()
324 .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
325 .await?
326 .ok_or(TransactionError::UnexpectedError(format!(
327 "Network with chain id {} not found",
328 evm_data.chain_id
329 )))?;
330
331 let network = EvmNetwork::try_from(network_model).map_err(|e| {
332 TransactionError::UnexpectedError(format!(
333 "Error converting network model to EvmNetwork: {e}"
334 ))
335 })?;
336
337 make_noop(&mut evm_data, &network, Some(self.provider())).await?;
338
339 let noop_count = tx.noop_count.unwrap_or(0) + 1;
340 let update_request = TransactionUpdateRequest {
341 network_data: Some(NetworkTransactionData::Evm(evm_data)),
342 noop_count: Some(noop_count),
343 status_reason: reason,
344 is_canceled: if is_cancellation {
345 Some(true)
346 } else {
347 tx.is_canceled
348 },
349 ..Default::default()
350 };
351 Ok(update_request)
352 }
353
354 async fn handle_submitted_state(
361 &self,
362 tx: TransactionRepoModel,
363 ) -> Result<TransactionRepoModel, TransactionError> {
364 if self.should_resubmit(&tx).await? {
365 if let Some(nonce_gap_detected) = self.detect_nonce_gap_ahead(&tx).await {
368 if nonce_gap_detected {
369 return self
371 .update_transaction_status_if_needed(tx, TransactionStatus::Submitted, None)
372 .await;
373 }
374 }
375
376 let resubmitted_tx = self.handle_resubmission(tx).await?;
377 return Ok(resubmitted_tx);
378 }
379
380 self.update_transaction_status_if_needed(tx, TransactionStatus::Submitted, None)
381 .await
382 }
383
384 async fn detect_nonce_gap_ahead(&self, tx: &TransactionRepoModel) -> Option<bool> {
398 let evm_data = match tx.network_data.get_evm_transaction_data() {
399 Ok(d) => d,
400 Err(_) => return None,
401 };
402 let tx_nonce = match evm_data.nonce {
403 Some(n) => n,
404 None => return None,
405 };
406
407 let on_chain_nonce = match self
408 .provider()
409 .get_transaction_count(&self.relayer().address)
410 .await
411 {
412 Ok(n) => n,
413 Err(e) => {
414 debug!(
415 tx_id = %tx.id,
416 error = %e,
417 "nonce gap check: failed to get on-chain nonce, skipping"
418 );
419 return None;
420 }
421 };
422
423 if tx_nonce <= on_chain_nonce {
425 return Some(false);
426 }
427
428 let scan_to = std::cmp::min(tx_nonce, on_chain_nonce + MAX_GAP_SCAN_RANGE);
432
433 let occupancy = match self
434 .transaction_repository()
435 .get_nonce_occupancy(&tx.relayer_id, on_chain_nonce, scan_to)
436 .await
437 {
438 Ok(o) => o,
439 Err(e) => {
440 debug!(
441 tx_id = %tx.id,
442 error = %e,
443 "nonce gap check: occupancy lookup failed, skipping"
444 );
445 return None;
446 }
447 };
448
449 let gap_nonces: Vec<u64> = occupancy
450 .into_iter()
451 .filter(|(_, status)| !status.as_ref().is_some_and(is_active_nonce_status))
452 .map(|(nonce, _)| nonce)
453 .collect();
454
455 if gap_nonces.is_empty() {
456 return Some(false);
458 }
459
460 warn!(
461 tx_id = %tx.id,
462 relayer_id = %tx.relayer_id,
463 tx_nonce = tx_nonce,
464 on_chain_nonce = on_chain_nonce,
465 gap_count = gap_nonces.len(),
466 gaps = ?gap_nonces,
467 "nonce gaps confirmed below tx, scheduling nonce health to fill"
468 );
469
470 if let Err(e) = self.schedule_relayer_nonce_health_job(tx).await {
471 warn!(
472 tx_id = %tx.id,
473 error = %e,
474 "failed to schedule nonce health job for nonce gap"
475 );
476 }
477
478 Some(true)
479 }
480
481 async fn handle_resubmission(
483 &self,
484 tx: TransactionRepoModel,
485 ) -> Result<TransactionRepoModel, TransactionError> {
486 debug!(
487 tx_id = %tx.id,
488 relayer_id = %tx.relayer_id,
489 status = ?tx.status,
490 "scheduling resubmit job for transaction"
491 );
492
493 let (should_noop, reason) = self.should_noop(&tx).await?;
495 let tx_to_process = if should_noop {
496 self.process_noop_transaction(&tx, reason).await?
497 } else {
498 tx
499 };
500
501 self.send_transaction_resubmit_job(&tx_to_process).await?;
502 Ok(tx_to_process)
503 }
504
505 async fn process_noop_transaction(
507 &self,
508 tx: &TransactionRepoModel,
509 reason: Option<String>,
510 ) -> Result<TransactionRepoModel, TransactionError> {
511 debug!(
512 tx_id = %tx.id,
513 relayer_id = %tx.relayer_id,
514 status = ?tx.status,
515 "preparing transaction NOOP before resubmission"
516 );
517 let update = self.prepare_noop_update_request(tx, false, reason).await?;
518 let updated_tx = self
519 .transaction_repository()
520 .partial_update(tx.id.clone(), update)
521 .await?;
522
523 let res = self.send_transaction_update_notification(&updated_tx).await;
524 if let Err(e) = res {
525 error!(
526 tx_id = %updated_tx.id,
527 relayer_id = %updated_tx.relayer_id,
528 status = ?updated_tx.status,
529 error = %e,
530 "sending transaction update notification failed for NOOP transaction"
531 );
532 }
533 Ok(updated_tx)
534 }
535
536 async fn handle_pending_state(
538 &self,
539 tx: TransactionRepoModel,
540 ) -> Result<TransactionRepoModel, TransactionError> {
541 let (should_noop, reason) = self.should_noop(&tx).await?;
542 if should_noop {
543 debug!(
546 tx_id = %tx.id,
547 relayer_id = %tx.relayer_id,
548 reason = %reason.as_ref().unwrap_or(&"unknown".to_string()),
549 "marking pending transaction as Failed (nonce not assigned, no NOOP needed)"
550 );
551 let update = TransactionUpdateRequest {
552 status: Some(TransactionStatus::Failed),
553 status_reason: reason,
554 ..Default::default()
555 };
556 let updated_tx = self
557 .transaction_repository()
558 .partial_update(tx.id.clone(), update)
559 .await?;
560
561 let res = self.send_transaction_update_notification(&updated_tx).await;
562 if let Err(e) = res {
563 error!(
564 tx_id = %updated_tx.id,
565 relayer_id = %updated_tx.relayer_id,
566 status = ?updated_tx.status,
567 error = %e,
568 "sending transaction update notification failed for Pending state NOOP"
569 );
570 }
571 return Ok(updated_tx);
572 }
573
574 let age = get_age_since_created(&tx)?;
576 if age > get_evm_pending_recovery_trigger_timeout() {
577 warn!(
578 tx_id = %tx.id,
579 relayer_id = %tx.relayer_id,
580 age_seconds = age.num_seconds(),
581 "transaction stuck in Pending, queuing prepare job"
582 );
583
584 self.send_transaction_request_job(&tx).await?;
586 }
587
588 Ok(tx)
589 }
590
591 async fn handle_mined_state(
593 &self,
594 tx: TransactionRepoModel,
595 ) -> Result<TransactionRepoModel, TransactionError> {
596 self.update_transaction_status_if_needed(tx, TransactionStatus::Mined, None)
597 .await
598 }
599
600 async fn handle_final_state(
602 &self,
603 tx: TransactionRepoModel,
604 status: TransactionStatus,
605 status_reason: Option<String>,
606 ) -> Result<TransactionRepoModel, TransactionError> {
607 self.update_transaction_status_if_needed(tx, status, status_reason)
608 .await
609 }
610
611 async fn mark_as_failed(
613 &self,
614 tx: TransactionRepoModel,
615 reason: String,
616 ) -> Result<TransactionRepoModel, TransactionError> {
617 warn!(
618 tx_id = %tx.id,
619 relayer_id = %tx.relayer_id,
620 reason = %reason,
621 "force-failing transaction due to circuit breaker"
622 );
623
624 let update = TransactionUpdateRequest {
625 status: Some(TransactionStatus::Failed),
626 status_reason: Some(reason),
627 ..Default::default()
628 };
629
630 let updated_tx = self
631 .transaction_repository()
632 .partial_update(tx.id.clone(), update)
633 .await?;
634
635 if let Err(e) = self.send_transaction_update_notification(&updated_tx).await {
637 error!(
638 tx_id = %updated_tx.id,
639 relayer_id = %updated_tx.relayer_id,
640 error = %e,
641 "failed to send notification for force-failed transaction"
642 );
643 }
644
645 Ok(updated_tx)
646 }
647
648 async fn reconcile_tx_nonce_state(
659 &self,
660 tx: &TransactionRepoModel,
661 ) -> Result<Option<TransactionRepoModel>, TransactionError> {
662 let evm_data = tx.network_data.get_evm_transaction_data()?;
663
664 let mut had_rpc_errors = false;
668
669 if let Some(ref hash) = evm_data.hash {
671 match self.provider().get_transaction_receipt(hash).await {
672 Ok(Some(_)) => {
673 debug!(
674 tx_id = %tx.id,
675 hash = %hash,
676 "nonce recovery: receipt found for current hash, deferring to normal flow"
677 );
678 return Ok(None);
679 }
680 Ok(None) => {
681 }
683 Err(e) => {
684 warn!(
685 tx_id = %tx.id,
686 hash = %hash,
687 error = %e,
688 "nonce recovery: error checking receipt for current hash"
689 );
690 had_rpc_errors = true;
691 }
692 }
693 }
694
695 if tx.hashes.len() > 1 {
697 match self.try_recover_with_historical_hashes(tx, &evm_data).await {
698 Ok(Some(recovered_tx)) => {
699 debug!(
700 tx_id = %tx.id,
701 "nonce recovery: recovered transaction via historical hash"
702 );
703 return Ok(Some(recovered_tx));
704 }
705 Ok(None) => {
706 }
708 Err(e) => {
709 warn!(
710 tx_id = %tx.id,
711 error = %e,
712 "nonce recovery: error during historical hash recovery"
713 );
714 had_rpc_errors = true;
715 }
716 }
717 }
718
719 if had_rpc_errors {
723 warn!(
724 tx_id = %tx.id,
725 "nonce recovery: skipping nonce comparison due to RPC errors during hash checks, deferring to normal flow"
726 );
727 return Ok(None);
728 }
729
730 let tx_nonce = match evm_data.nonce {
731 Some(n) => n,
732 None => {
733 return Ok(None);
735 }
736 };
737
738 let on_chain_nonce = self
739 .provider()
740 .get_transaction_count(&self.relayer().address)
741 .await
742 .map_err(|e| {
743 TransactionError::UnexpectedError(format!(
744 "Failed to get on-chain nonce for recovery: {e}"
745 ))
746 })?;
747
748 if on_chain_nonce > tx_nonce {
749 let reason = format!(
751 "Nonce {tx_nonce} consumed externally (on-chain nonce: {on_chain_nonce}). \
752 No matching transaction hash found on-chain."
753 );
754 warn!(
755 tx_id = %tx.id,
756 relayer_id = %tx.relayer_id,
757 tx_nonce = tx_nonce,
758 on_chain_nonce = on_chain_nonce,
759 "nonce recovery: nonce consumed externally, marking as Failed"
760 );
761
762 let updated_tx = self
763 .update_transaction_status(tx.clone(), TransactionStatus::Failed, Some(reason))
764 .await?;
765
766 if let Err(e) = self.schedule_relayer_nonce_health_job(tx).await {
771 warn!(
772 tx_id = %tx.id,
773 error = %e,
774 "nonce recovery: failed to schedule nonce health after external consumption"
775 );
776 }
777
778 return Ok(Some(updated_tx));
779 }
780
781 debug!(
783 tx_id = %tx.id,
784 tx_nonce = tx_nonce,
785 on_chain_nonce = on_chain_nonce,
786 "nonce recovery: on-chain nonce not past tx nonce, deferring to normal flow"
787 );
788 Ok(None)
789 }
790
791 async fn handle_circuit_breaker_safely(
803 &self,
804 tx: TransactionRepoModel,
805 ctx: &StatusCheckContext,
806 ) -> Result<TransactionRepoModel, TransactionError> {
807 let reason = format!(
808 "Transaction status monitoring failed after {} consecutive errors (total: {}). \
809 Last status: {:?}.",
810 ctx.consecutive_failures, ctx.total_failures, tx.status
811 );
812
813 match tx.status {
814 TransactionStatus::Pending => {
815 debug!(
817 tx_id = %tx.id,
818 relayer_id = %tx.relayer_id,
819 "circuit breaker: Pending transaction (no nonce) - safe to mark as Failed"
820 );
821 self.mark_as_failed(tx, reason).await
822 }
823 TransactionStatus::Sent => {
824 let has_nonce = tx
828 .network_data
829 .get_evm_transaction_data()
830 .map(|d| d.nonce.is_some())
831 .unwrap_or(false);
832
833 if has_nonce {
834 warn!(
835 tx_id = %tx.id,
836 relayer_id = %tx.relayer_id,
837 "circuit breaker: Sent transaction with nonce assigned - triggering NOOP to clear nonce slot"
838 );
839 let noop_reason = Some(format!(
840 "{reason}. Replacing with NOOP to clear nonce slot (Sent state with assigned nonce)."
841 ));
842 let updated_tx = self.process_noop_transaction(&tx, noop_reason).await?;
843 self.send_transaction_resubmit_job(&updated_tx).await?;
847 Ok(updated_tx)
848 } else {
849 debug!(
851 tx_id = %tx.id,
852 relayer_id = %tx.relayer_id,
853 "circuit breaker: Sent transaction without nonce - safe to mark as Failed"
854 );
855 self.mark_as_failed(tx, reason).await
856 }
857 }
858 TransactionStatus::Submitted => {
859 warn!(
865 tx_id = %tx.id,
866 relayer_id = %tx.relayer_id,
867 "circuit breaker: Submitted transaction - triggering NOOP to safely clear nonce"
868 );
869 let noop_reason = Some(format!(
870 "{reason}. Replacing with NOOP to clear nonce slot."
871 ));
872 let updated_tx = self.process_noop_transaction(&tx, noop_reason).await?;
873 self.send_transaction_resubmit_job(&updated_tx).await?;
874 Ok(updated_tx)
875 }
876 _ => {
877 debug!(
879 tx_id = %tx.id,
880 relayer_id = %tx.relayer_id,
881 status = ?tx.status,
882 "circuit breaker: unexpected status, returning transaction unchanged"
883 );
884 Ok(tx)
885 }
886 }
887 }
888
889 pub async fn handle_status_impl(
894 &self,
895 tx: TransactionRepoModel,
896 context: Option<StatusCheckContext>,
897 ) -> Result<TransactionRepoModel, TransactionError> {
898 debug!(
899 tx_id = %tx.id,
900 relayer_id = %tx.relayer_id,
901 status = ?tx.status,
902 "checking transaction status"
903 );
904
905 if is_final_state(&tx.status) {
907 debug!(
908 tx_id = %tx.id,
909 relayer_id = %tx.relayer_id,
910 status = ?tx.status,
911 "transaction already in final state"
912 );
913 return Ok(tx);
914 }
915
916 if let Some(ref ctx) = context {
920 let is_noop_tx = tx
921 .network_data
922 .get_evm_transaction_data()
923 .map(|data| is_noop(&data))
924 .unwrap_or(false);
925
926 if ctx.should_force_finalize() && !is_noop_tx {
927 warn!(
928 tx_id = %tx.id,
929 consecutive_failures = ctx.consecutive_failures,
930 total_failures = ctx.total_failures,
931 max_consecutive = ctx.max_consecutive_failures,
932 status = ?tx.status,
933 "circuit breaker triggered - handling safely based on transaction state"
934 );
935 return self.handle_circuit_breaker_safely(tx, ctx).await;
936 }
937
938 if ctx.should_force_finalize() && is_noop_tx {
939 debug!(
940 tx_id = %tx.id,
941 consecutive_failures = ctx.consecutive_failures,
942 relayer_id = %tx.relayer_id,
943 "circuit breaker would trigger but transaction is NOOP - continuing with normal status logic"
944 );
945 }
946 }
947
948 if let Some(ref ctx) = context {
952 if let Some(ref metadata) = ctx.job_metadata {
953 if let Some(hint) = metadata.get(TX_NONCE_RECONCILE_TRIGGER) {
954 debug!(
955 tx_id = %tx.id,
956 hint = %hint,
957 "nonce recovery hint detected - performing nonce reconciliation"
958 );
959 match self.reconcile_tx_nonce_state(&tx).await {
960 Ok(Some(recovered_tx)) => {
961 return Ok(recovered_tx);
962 }
963 Ok(None) => {
964 debug!(
966 tx_id = %tx.id,
967 "nonce recovery did not resolve transaction, continuing normal flow"
968 );
969 }
970 Err(e) => {
971 warn!(
973 tx_id = %tx.id,
974 error = %e,
975 "nonce recovery failed, falling through to normal status flow"
976 );
977 }
978 }
979 }
980 }
981 }
982
983 let status = self.check_transaction_status(&tx).await?;
988
989 debug!(
990 tx_id = %tx.id,
991 previous_status = ?tx.status,
992 new_status = ?status,
993 relayer_id = %tx.relayer_id,
994 "transaction status check completed"
995 );
996
997 let tx = if status != tx.status {
1001 debug!(
1002 tx_id = %tx.id,
1003 old_status = ?tx.status,
1004 new_status = ?status,
1005 relayer_id = %tx.relayer_id,
1006 "status changed during check, reloading transaction from DB to ensure fresh data"
1007 );
1008 self.transaction_repository()
1009 .get_by_id(tx.id.clone())
1010 .await?
1011 } else {
1012 tx
1013 };
1014
1015 if is_too_early_to_resubmit(&tx)? && is_pending_transaction(&status) {
1020 return self
1022 .update_transaction_status_if_needed(tx, status, None)
1023 .await;
1024 }
1025
1026 match status {
1028 TransactionStatus::Pending => self.handle_pending_state(tx).await,
1029 TransactionStatus::Sent => self.handle_sent_state(tx).await,
1030 TransactionStatus::Submitted => self.handle_submitted_state(tx).await,
1031 TransactionStatus::Mined => self.handle_mined_state(tx).await,
1032 TransactionStatus::Failed => {
1033 let status_reason = if tx.status != TransactionStatus::Failed {
1036 Some("Transaction reverted on-chain (receipt status: failed)".to_string())
1037 } else {
1038 None
1039 };
1040 self.handle_final_state(tx, status, status_reason).await
1041 }
1042 TransactionStatus::Confirmed
1043 | TransactionStatus::Expired
1044 | TransactionStatus::Canceled => self.handle_final_state(tx, status, None).await,
1045 }
1046 }
1047
1048 async fn handle_sent_state(
1050 &self,
1051 tx: TransactionRepoModel,
1052 ) -> Result<TransactionRepoModel, TransactionError> {
1053 debug!(
1054 tx_id = %tx.id,
1055 relayer_id = %tx.relayer_id,
1056 "handling Sent state"
1057 );
1058
1059 let (should_noop, reason) = self.should_noop(&tx).await?;
1061 if should_noop {
1062 debug!(
1063 tx_id = %tx.id,
1064 relayer_id = %tx.relayer_id,
1065 "preparing NOOP for sent transaction"
1066 );
1067 let update = self.prepare_noop_update_request(&tx, false, reason).await?;
1068 let updated_tx = self
1069 .transaction_repository()
1070 .partial_update(tx.id.clone(), update)
1071 .await?;
1072
1073 self.send_transaction_submit_job(&updated_tx).await?;
1074 let res = self.send_transaction_update_notification(&updated_tx).await;
1075 if let Err(e) = res {
1076 error!(
1077 tx_id = %updated_tx.id,
1078 relayer_id = %updated_tx.relayer_id,
1079 status = ?updated_tx.status,
1080 error = %e,
1081 "sending transaction update notification failed for Sent state NOOP"
1082 );
1083 }
1084 return Ok(updated_tx);
1085 }
1086
1087 let age_since_sent = get_age_since_status_change(&tx)?;
1090
1091 if age_since_sent > get_evm_resend_timeout() {
1092 warn!(
1093 tx_id = %tx.id,
1094 relayer_id = %tx.relayer_id,
1095 age_seconds = age_since_sent.num_seconds(),
1096 "transaction stuck in Sent, queuing resubmit job with repricing"
1097 );
1098
1099 self.send_transaction_resubmit_job(&tx).await?;
1101 }
1102
1103 self.update_transaction_status_if_needed(tx, TransactionStatus::Sent, None)
1104 .await
1105 }
1106
1107 fn should_try_hash_recovery(
1114 &self,
1115 tx: &TransactionRepoModel,
1116 ) -> Result<bool, TransactionError> {
1117 if tx.status != TransactionStatus::Submitted {
1119 return Ok(false);
1120 }
1121
1122 if tx.hashes.len() <= 1 {
1124 return Ok(false);
1125 }
1126
1127 let age = get_age_of_sent_at(tx)?;
1129 let min_age_for_recovery = get_evm_min_age_for_hash_recovery();
1130
1131 if age < min_age_for_recovery {
1132 return Ok(false);
1133 }
1134
1135 if tx.hashes.len() < EVM_MIN_HASHES_FOR_RECOVERY {
1138 return Ok(false);
1139 }
1140
1141 Ok(true)
1142 }
1143
1144 async fn try_recover_with_historical_hashes(
1153 &self,
1154 tx: &TransactionRepoModel,
1155 evm_data: &crate::models::EvmTransactionData,
1156 ) -> Result<Option<TransactionRepoModel>, TransactionError> {
1157 warn!(
1158 tx_id = %tx.id,
1159 relayer_id = %tx.relayer_id,
1160 current_hash = ?evm_data.hash,
1161 total_hashes = %tx.hashes.len(),
1162 "attempting hash recovery - checking historical hashes"
1163 );
1164
1165 for (idx, historical_hash) in tx.hashes.iter().rev().enumerate() {
1167 if Some(historical_hash) == evm_data.hash.as_ref() {
1169 continue;
1170 }
1171
1172 debug!(
1173 tx_id = %tx.id,
1174 relayer_id = %tx.relayer_id,
1175 hash = %historical_hash,
1176 index = %idx,
1177 "checking historical hash"
1178 );
1179
1180 match self
1182 .provider()
1183 .get_transaction_receipt(historical_hash)
1184 .await
1185 {
1186 Ok(Some(receipt)) => {
1187 warn!(
1188 tx_id = %tx.id,
1189 relayer_id = %tx.relayer_id,
1190 mined_hash = %historical_hash,
1191 wrong_hash = ?evm_data.hash,
1192 block_number = ?receipt.block_number,
1193 "RECOVERED: found mined transaction with historical hash - correcting database"
1194 );
1195
1196 let updated_tx = self
1199 .update_transaction_with_corrected_hash(
1200 tx,
1201 evm_data,
1202 historical_hash,
1203 TransactionStatus::Mined,
1204 )
1205 .await?;
1206
1207 return Ok(Some(updated_tx));
1208 }
1209 Ok(None) => {
1210 continue;
1212 }
1213 Err(e) => {
1214 warn!(
1216 tx_id = %tx.id,
1217 relayer_id = %tx.relayer_id,
1218 hash = %historical_hash,
1219 error = %e,
1220 "error checking historical hash, continuing to next"
1221 );
1222 continue;
1223 }
1224 }
1225 }
1226
1227 debug!(
1229 tx_id = %tx.id,
1230 relayer_id = %tx.relayer_id,
1231 "hash recovery completed - no historical hashes found on-chain"
1232 );
1233 Ok(None)
1234 }
1235
1236 async fn update_transaction_with_corrected_hash(
1240 &self,
1241 tx: &TransactionRepoModel,
1242 evm_data: &crate::models::EvmTransactionData,
1243 correct_hash: &str,
1244 status: TransactionStatus,
1245 ) -> Result<TransactionRepoModel, TransactionError> {
1246 let mut corrected_data = evm_data.clone();
1247 corrected_data.hash = Some(correct_hash.to_string());
1248
1249 let updated_tx = self
1250 .transaction_repository()
1251 .partial_update(
1252 tx.id.clone(),
1253 TransactionUpdateRequest {
1254 network_data: Some(NetworkTransactionData::Evm(corrected_data)),
1255 status: Some(status),
1256 ..Default::default()
1257 },
1258 )
1259 .await?;
1260
1261 if let Err(e) = self.send_transaction_update_notification(&updated_tx).await {
1263 error!(
1264 tx_id = %updated_tx.id,
1265 relayer_id = %updated_tx.relayer_id,
1266 error = %e,
1267 "failed to send notification for hash recovery"
1268 );
1269 }
1270
1271 Ok(updated_tx)
1272 }
1273}
1274
1275#[cfg(test)]
1276mod tests {
1277 use crate::{
1278 config::{EvmNetworkConfig, NetworkConfigCommon},
1279 domain::transaction::evm::{EvmRelayerTransaction, MockPriceCalculatorTrait},
1280 jobs::MockJobProducerTrait,
1281 models::{
1282 evm::Speed, EvmTransactionData, NetworkConfigData, NetworkRepoModel,
1283 NetworkTransactionData, NetworkType, RelayerEvmPolicy, RelayerNetworkPolicy,
1284 RelayerRepoModel, RpcConfig, TransactionReceipt, TransactionRepoModel,
1285 TransactionStatus, U256,
1286 },
1287 repositories::{
1288 MockNetworkRepository, MockRelayerRepository, MockTransactionCounterTrait,
1289 MockTransactionRepository,
1290 },
1291 services::{provider::MockEvmProviderTrait, signer::MockSigner},
1292 };
1293 use alloy::{
1294 consensus::{Eip658Value, Receipt, ReceiptWithBloom},
1295 network::AnyReceiptEnvelope,
1296 primitives::{b256, Address, BlockHash, Bloom, TxHash},
1297 };
1298 use chrono::{Duration, Utc};
1299 use std::sync::Arc;
1300
1301 pub struct TestMocks {
1303 pub provider: MockEvmProviderTrait,
1304 pub relayer_repo: MockRelayerRepository,
1305 pub network_repo: MockNetworkRepository,
1306 pub tx_repo: MockTransactionRepository,
1307 pub job_producer: MockJobProducerTrait,
1308 pub signer: MockSigner,
1309 pub counter: MockTransactionCounterTrait,
1310 pub price_calc: MockPriceCalculatorTrait,
1311 }
1312
1313 pub fn default_test_mocks() -> TestMocks {
1316 TestMocks {
1317 provider: MockEvmProviderTrait::new(),
1318 relayer_repo: MockRelayerRepository::new(),
1319 network_repo: MockNetworkRepository::new(),
1320 tx_repo: MockTransactionRepository::new(),
1321 job_producer: MockJobProducerTrait::new(),
1322 signer: MockSigner::new(),
1323 counter: MockTransactionCounterTrait::new(),
1324 price_calc: MockPriceCalculatorTrait::new(),
1325 }
1326 }
1327
1328 pub fn default_test_mocks_with_network() -> TestMocks {
1330 let mut mocks = default_test_mocks();
1331 mocks
1333 .network_repo
1334 .expect_get_by_chain_id()
1335 .returning(|network_type, chain_id| {
1336 if network_type == NetworkType::Evm && chain_id == 1 {
1337 Ok(Some(create_test_network_model()))
1338 } else {
1339 Ok(None)
1340 }
1341 });
1342 mocks
1343 }
1344
1345 pub fn create_test_network_model() -> NetworkRepoModel {
1347 let evm_config = EvmNetworkConfig {
1348 common: NetworkConfigCommon {
1349 network: "mainnet".to_string(),
1350 from: None,
1351 rpc_urls: Some(vec![RpcConfig::new("https://rpc.example.com".to_string())]),
1352 explorer_urls: Some(vec!["https://explorer.example.com".to_string()]),
1353 average_blocktime_ms: Some(12000),
1354 is_testnet: Some(false),
1355 tags: Some(vec!["mainnet".to_string()]),
1356 },
1357 chain_id: Some(1),
1358 required_confirmations: Some(12),
1359 features: Some(vec!["eip1559".to_string()]),
1360 symbol: Some("ETH".to_string()),
1361 gas_price_cache: None,
1362 };
1363 NetworkRepoModel {
1364 id: "evm:mainnet".to_string(),
1365 name: "mainnet".to_string(),
1366 network_type: NetworkType::Evm,
1367 config: NetworkConfigData::Evm(evm_config),
1368 }
1369 }
1370
1371 pub fn create_test_no_mempool_network_model() -> NetworkRepoModel {
1373 let evm_config = EvmNetworkConfig {
1374 common: NetworkConfigCommon {
1375 network: "arbitrum".to_string(),
1376 from: None,
1377 rpc_urls: Some(vec![crate::models::RpcConfig::new(
1378 "https://arb-rpc.example.com".to_string(),
1379 )]),
1380 explorer_urls: Some(vec!["https://arb-explorer.example.com".to_string()]),
1381 average_blocktime_ms: Some(1000),
1382 is_testnet: Some(false),
1383 tags: Some(vec![
1384 "arbitrum".to_string(),
1385 "rollup".to_string(),
1386 "no-mempool".to_string(),
1387 ]),
1388 },
1389 chain_id: Some(42161),
1390 required_confirmations: Some(12),
1391 features: Some(vec!["eip1559".to_string()]),
1392 symbol: Some("ETH".to_string()),
1393 gas_price_cache: None,
1394 };
1395 NetworkRepoModel {
1396 id: "evm:arbitrum".to_string(),
1397 name: "arbitrum".to_string(),
1398 network_type: NetworkType::Evm,
1399 config: NetworkConfigData::Evm(evm_config),
1400 }
1401 }
1402
1403 pub fn make_test_transaction(status: TransactionStatus) -> TransactionRepoModel {
1407 TransactionRepoModel {
1408 id: "test-tx-id".to_string(),
1409 relayer_id: "test-relayer-id".to_string(),
1410 status,
1411 status_reason: None,
1412 created_at: Utc::now().to_rfc3339(),
1413 sent_at: None,
1414 confirmed_at: None,
1415 valid_until: None,
1416 delete_at: None,
1417 network_type: NetworkType::Evm,
1418 network_data: NetworkTransactionData::Evm(EvmTransactionData {
1419 chain_id: 1,
1420 from: "0xSender".to_string(),
1421 to: Some("0xRecipient".to_string()),
1422 value: U256::from(0),
1423 data: Some("0xData".to_string()),
1424 gas_limit: Some(21000),
1425 gas_price: Some(20000000000),
1426 max_fee_per_gas: None,
1427 max_priority_fee_per_gas: None,
1428 nonce: None,
1429 signature: None,
1430 hash: None,
1431 speed: Some(Speed::Fast),
1432 raw: None,
1433 }),
1434 priced_at: None,
1435 hashes: Vec::new(),
1436 noop_count: None,
1437 is_canceled: Some(false),
1438 metadata: None,
1439 }
1440 }
1441
1442 pub fn make_test_evm_relayer_transaction(
1445 relayer: RelayerRepoModel,
1446 mocks: TestMocks,
1447 ) -> EvmRelayerTransaction<
1448 MockEvmProviderTrait,
1449 MockRelayerRepository,
1450 MockNetworkRepository,
1451 MockTransactionRepository,
1452 MockJobProducerTrait,
1453 MockSigner,
1454 MockTransactionCounterTrait,
1455 MockPriceCalculatorTrait,
1456 > {
1457 EvmRelayerTransaction::new(
1458 relayer,
1459 mocks.provider,
1460 Arc::new(mocks.relayer_repo),
1461 Arc::new(mocks.network_repo),
1462 Arc::new(mocks.tx_repo),
1463 Arc::new(mocks.counter),
1464 Arc::new(mocks.job_producer),
1465 mocks.price_calc,
1466 mocks.signer,
1467 )
1468 .unwrap()
1469 }
1470
1471 fn create_test_relayer() -> RelayerRepoModel {
1472 RelayerRepoModel {
1473 id: "test-relayer-id".to_string(),
1474 name: "Test Relayer".to_string(),
1475 paused: false,
1476 system_disabled: false,
1477 network: "test_network".to_string(),
1478 network_type: NetworkType::Evm,
1479 policies: RelayerNetworkPolicy::Evm(RelayerEvmPolicy::default()),
1480 signer_id: "test_signer".to_string(),
1481 address: "0x".to_string(),
1482 notification_id: None,
1483 custom_rpc_urls: None,
1484 ..Default::default()
1485 }
1486 }
1487
1488 fn make_mock_receipt(status: bool, block_number: Option<u64>) -> TransactionReceipt {
1489 let tx_hash = TxHash::from(b256!(
1491 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
1492 ));
1493 let block_hash = BlockHash::from(b256!(
1494 "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
1495 ));
1496 let from_address = Address::from([0x11; 20]);
1497
1498 TransactionReceipt {
1499 inner: alloy::rpc::types::TransactionReceipt {
1500 inner: AnyReceiptEnvelope {
1501 inner: ReceiptWithBloom {
1502 receipt: Receipt {
1503 status: Eip658Value::Eip658(status), cumulative_gas_used: 0,
1505 logs: vec![],
1506 },
1507 logs_bloom: Bloom::ZERO,
1508 },
1509 r#type: 0, },
1511 transaction_hash: tx_hash,
1512 transaction_index: Some(0),
1513 block_hash: block_number.map(|_| block_hash), block_number,
1515 gas_used: 21000,
1516 effective_gas_price: 1000,
1517 blob_gas_used: None,
1518 blob_gas_price: None,
1519 from: from_address,
1520 to: None,
1521 contract_address: None,
1522 },
1523 other: Default::default(),
1524 }
1525 }
1526
1527 mod check_transaction_status_tests {
1529 use super::*;
1530
1531 #[tokio::test]
1532 async fn test_not_mined() {
1533 let mut mocks = default_test_mocks();
1534 let relayer = create_test_relayer();
1535 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1536
1537 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1539 evm_data.hash = Some("0xFakeHash".to_string());
1540 }
1541
1542 mocks
1544 .provider
1545 .expect_get_transaction_receipt()
1546 .returning(|_| Box::pin(async { Ok(None) }));
1547
1548 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1549
1550 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1551 assert_eq!(status, TransactionStatus::Submitted);
1552 }
1553
1554 #[tokio::test]
1555 async fn test_mined_but_not_confirmed() {
1556 let mut mocks = default_test_mocks();
1557 let relayer = create_test_relayer();
1558 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1559
1560 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1561 evm_data.hash = Some("0xFakeHash".to_string());
1562 }
1563
1564 mocks
1566 .provider
1567 .expect_get_transaction_receipt()
1568 .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) }));
1569
1570 mocks
1572 .provider
1573 .expect_get_block_number()
1574 .return_once(|| Box::pin(async { Ok(100) }));
1575
1576 mocks
1578 .network_repo
1579 .expect_get_by_chain_id()
1580 .returning(|_, _| Ok(Some(create_test_network_model())));
1581
1582 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1583
1584 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1585 assert_eq!(status, TransactionStatus::Mined);
1586 }
1587
1588 #[tokio::test]
1589 async fn test_confirmed() {
1590 let mut mocks = default_test_mocks();
1591 let relayer = create_test_relayer();
1592 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1593
1594 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1595 evm_data.hash = Some("0xFakeHash".to_string());
1596 }
1597
1598 mocks
1600 .provider
1601 .expect_get_transaction_receipt()
1602 .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) }));
1603
1604 mocks
1606 .provider
1607 .expect_get_block_number()
1608 .return_once(|| Box::pin(async { Ok(113) }));
1609
1610 mocks
1612 .network_repo
1613 .expect_get_by_chain_id()
1614 .returning(|_, _| Ok(Some(create_test_network_model())));
1615
1616 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1617
1618 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1619 assert_eq!(status, TransactionStatus::Confirmed);
1620 }
1621
1622 #[tokio::test]
1623 async fn test_failed() {
1624 let mut mocks = default_test_mocks();
1625 let relayer = create_test_relayer();
1626 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1627
1628 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1629 evm_data.hash = Some("0xFakeHash".to_string());
1630 }
1631
1632 mocks
1634 .provider
1635 .expect_get_transaction_receipt()
1636 .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(false, Some(100)))) }));
1637
1638 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1639
1640 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1641 assert_eq!(status, TransactionStatus::Failed);
1642 }
1643 }
1644
1645 mod should_resubmit_tests {
1647 use super::*;
1648 use crate::models::TransactionError;
1649
1650 #[tokio::test]
1651 async fn test_should_resubmit_true() {
1652 let mut mocks = default_test_mocks();
1653 let relayer = create_test_relayer();
1654
1655 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1657 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1658
1659 mocks
1661 .network_repo
1662 .expect_get_by_chain_id()
1663 .returning(|_, _| Ok(Some(create_test_network_model())));
1664
1665 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1666 let res = evm_transaction.should_resubmit(&tx).await.unwrap();
1667 assert!(res, "Transaction should be resubmitted after timeout.");
1668 }
1669
1670 #[tokio::test]
1671 async fn test_should_resubmit_false() {
1672 let mut mocks = default_test_mocks();
1673 let relayer = create_test_relayer();
1674
1675 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1677 tx.sent_at = Some(Utc::now().to_rfc3339());
1678
1679 mocks
1681 .network_repo
1682 .expect_get_by_chain_id()
1683 .returning(|_, _| Ok(Some(create_test_network_model())));
1684
1685 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1686 let res = evm_transaction.should_resubmit(&tx).await.unwrap();
1687 assert!(!res, "Transaction should not be resubmitted immediately.");
1688 }
1689
1690 #[tokio::test]
1691 async fn test_should_resubmit_true_for_no_mempool_network() {
1692 let mut mocks = default_test_mocks();
1693 let relayer = create_test_relayer();
1694
1695 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1697 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1698
1699 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1701 evm_data.chain_id = 42161; }
1703
1704 mocks
1706 .network_repo
1707 .expect_get_by_chain_id()
1708 .returning(|_, _| Ok(Some(create_test_no_mempool_network_model())));
1709
1710 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1711 let res = evm_transaction.should_resubmit(&tx).await.unwrap();
1712 assert!(
1713 res,
1714 "Transaction should be resubmitted for no-mempool networks."
1715 );
1716 }
1717
1718 #[tokio::test]
1719 async fn test_should_resubmit_network_not_found() {
1720 let mut mocks = default_test_mocks();
1721 let relayer = create_test_relayer();
1722
1723 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1724 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1725
1726 mocks
1728 .network_repo
1729 .expect_get_by_chain_id()
1730 .returning(|_, _| Ok(None));
1731
1732 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1733 let result = evm_transaction.should_resubmit(&tx).await;
1734
1735 assert!(
1736 result.is_err(),
1737 "should_resubmit should return error when network not found"
1738 );
1739 let error = result.unwrap_err();
1740 match error {
1741 TransactionError::UnexpectedError(msg) => {
1742 assert!(msg.contains("Network with chain id 1 not found"));
1743 }
1744 _ => panic!("Expected UnexpectedError for network not found"),
1745 }
1746 }
1747
1748 #[tokio::test]
1749 async fn test_should_resubmit_network_conversion_error() {
1750 let mut mocks = default_test_mocks();
1751 let relayer = create_test_relayer();
1752
1753 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1754 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1755
1756 let invalid_evm_config = EvmNetworkConfig {
1758 common: NetworkConfigCommon {
1759 network: "invalid-network".to_string(),
1760 from: None,
1761 rpc_urls: Some(vec![crate::models::RpcConfig::new(
1762 "https://rpc.example.com".to_string(),
1763 )]),
1764 explorer_urls: Some(vec!["https://explorer.example.com".to_string()]),
1765 average_blocktime_ms: Some(12000),
1766 is_testnet: Some(false),
1767 tags: Some(vec!["testnet".to_string()]),
1768 },
1769 chain_id: None, required_confirmations: Some(12),
1771 features: Some(vec!["eip1559".to_string()]),
1772 symbol: Some("ETH".to_string()),
1773 gas_price_cache: None,
1774 };
1775 let invalid_network = NetworkRepoModel {
1776 id: "evm:invalid".to_string(),
1777 name: "invalid-network".to_string(),
1778 network_type: NetworkType::Evm,
1779 config: NetworkConfigData::Evm(invalid_evm_config),
1780 };
1781
1782 mocks
1784 .network_repo
1785 .expect_get_by_chain_id()
1786 .returning(move |_, _| Ok(Some(invalid_network.clone())));
1787
1788 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1789 let result = evm_transaction.should_resubmit(&tx).await;
1790
1791 assert!(
1792 result.is_err(),
1793 "should_resubmit should return error when network conversion fails"
1794 );
1795 let error = result.unwrap_err();
1796 match error {
1797 TransactionError::UnexpectedError(msg) => {
1798 assert!(msg.contains("Error converting network model to EvmNetwork"));
1799 }
1800 _ => panic!("Expected UnexpectedError for network conversion failure"),
1801 }
1802 }
1803 }
1804
1805 mod should_noop_tests {
1807 use super::*;
1808
1809 #[tokio::test]
1810 async fn test_expired_transaction_triggers_noop() {
1811 let mut mocks = default_test_mocks();
1812 let relayer = create_test_relayer();
1813
1814 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1815 tx.valid_until = Some((Utc::now() - Duration::seconds(10)).to_rfc3339());
1817
1818 mocks
1820 .network_repo
1821 .expect_get_by_chain_id()
1822 .returning(|_, _| Ok(Some(create_test_network_model())));
1823
1824 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1825 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
1826 assert!(res, "Expired transaction should be replaced with a NOOP.");
1827 assert!(
1828 reason.is_some(),
1829 "Reason should be provided for expired transaction"
1830 );
1831 assert!(
1832 reason.unwrap().contains("expired"),
1833 "Reason should mention expiration"
1834 );
1835 }
1836
1837 #[tokio::test]
1838 async fn test_too_many_noop_attempts_returns_false() {
1839 let mocks = default_test_mocks();
1840 let relayer = create_test_relayer();
1841
1842 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1843 tx.noop_count = Some(51); let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1846 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
1847 assert!(
1848 !res,
1849 "Transaction with too many NOOP attempts should not be replaced."
1850 );
1851 assert!(
1852 reason.is_none(),
1853 "Reason should not be provided when should_noop is false"
1854 );
1855 }
1856
1857 #[tokio::test]
1858 async fn test_already_noop_returns_false() {
1859 let mut mocks = default_test_mocks();
1860 let relayer = create_test_relayer();
1861
1862 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1863 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1865 evm_data.to = None;
1866 evm_data.value = U256::from(0);
1867 }
1868
1869 mocks
1870 .network_repo
1871 .expect_get_by_chain_id()
1872 .returning(|_, _| Ok(Some(create_test_network_model())));
1873
1874 mocks.provider.expect_get_block_by_number().returning(|| {
1876 Box::pin(async {
1877 use alloy::{network::AnyRpcBlock, rpc::types::Block};
1878 let mut block: Block = Block::default();
1879 block.header.gas_limit = 30_000_000u64;
1880 Ok(AnyRpcBlock::from(block))
1881 })
1882 });
1883
1884 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1885 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
1886 assert!(
1887 !res,
1888 "Transaction that is already a NOOP should not be replaced."
1889 );
1890 assert!(
1891 reason.is_none(),
1892 "Reason should not be provided when should_noop is false"
1893 );
1894 }
1895
1896 #[tokio::test]
1897 async fn test_rollup_with_too_many_attempts_triggers_noop() {
1898 let mut mocks = default_test_mocks();
1899 let relayer = create_test_relayer();
1900
1901 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1902 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1904 evm_data.chain_id = 42161; }
1906 tx.hashes = vec!["0xHash1".to_string(); 51];
1908
1909 mocks
1911 .network_repo
1912 .expect_get_by_chain_id()
1913 .returning(|_, _| Ok(Some(create_test_no_mempool_network_model())));
1914
1915 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1916 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
1917 assert!(
1918 res,
1919 "Rollup transaction with too many attempts should be replaced with NOOP."
1920 );
1921 assert!(
1922 reason.is_some(),
1923 "Reason should be provided for rollup transaction"
1924 );
1925 assert!(
1926 reason.unwrap().contains("too many attempts"),
1927 "Reason should mention too many attempts"
1928 );
1929 }
1930
1931 #[tokio::test]
1932 async fn test_pending_state_timeout_triggers_noop() {
1933 let mut mocks = default_test_mocks();
1934 let relayer = create_test_relayer();
1935
1936 let mut tx = make_test_transaction(TransactionStatus::Pending);
1937 tx.created_at = (Utc::now() - Duration::minutes(3)).to_rfc3339();
1939
1940 mocks
1941 .network_repo
1942 .expect_get_by_chain_id()
1943 .returning(|_, _| Ok(Some(create_test_network_model())));
1944
1945 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1946 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
1947 assert!(
1948 res,
1949 "Pending transaction stuck for >2 minutes should be replaced with NOOP."
1950 );
1951 assert!(
1952 reason.is_some(),
1953 "Reason should be provided for pending timeout"
1954 );
1955 assert!(
1956 reason.unwrap().contains("Pending state"),
1957 "Reason should mention Pending state"
1958 );
1959 }
1960
1961 #[tokio::test]
1962 async fn test_valid_transaction_returns_false() {
1963 let mut mocks = default_test_mocks();
1964 let relayer = create_test_relayer();
1965
1966 let tx = make_test_transaction(TransactionStatus::Submitted);
1967 mocks
1970 .network_repo
1971 .expect_get_by_chain_id()
1972 .returning(|_, _| Ok(Some(create_test_network_model())));
1973
1974 mocks.provider.expect_get_block_by_number().returning(|| {
1976 Box::pin(async {
1977 use alloy::{network::AnyRpcBlock, rpc::types::Block};
1978 let mut block: Block = Block::default();
1979 block.header.gas_limit = 30_000_000u64;
1980 Ok(AnyRpcBlock::from(block))
1981 })
1982 });
1983
1984 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1985 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
1986 assert!(!res, "Valid transaction should not be replaced with NOOP.");
1987 assert!(
1988 reason.is_none(),
1989 "Reason should not be provided when should_noop is false"
1990 );
1991 }
1992 }
1993
1994 mod update_transaction_status_tests {
1996 use super::*;
1997
1998 #[tokio::test]
1999 async fn test_no_update_when_status_is_same() {
2000 let mocks = default_test_mocks();
2002 let relayer = create_test_relayer();
2003 let tx = make_test_transaction(TransactionStatus::Submitted);
2004 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2005
2006 let updated_tx = evm_transaction
2009 .update_transaction_status_if_needed(tx.clone(), TransactionStatus::Submitted, None)
2010 .await
2011 .unwrap();
2012 assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2013 assert_eq!(updated_tx.id, tx.id);
2014 }
2015
2016 #[tokio::test]
2017 async fn test_updates_when_status_differs() {
2018 let mut mocks = default_test_mocks();
2019 let relayer = create_test_relayer();
2020 let tx = make_test_transaction(TransactionStatus::Submitted);
2021
2022 mocks
2024 .tx_repo
2025 .expect_partial_update()
2026 .returning(|_, update| {
2027 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2028 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2029 Ok(updated_tx)
2030 });
2031
2032 mocks
2034 .job_producer
2035 .expect_produce_send_notification_job()
2036 .returning(|_, _| Box::pin(async { Ok(()) }));
2037
2038 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2039 let updated_tx = evm_transaction
2040 .update_transaction_status_if_needed(tx.clone(), TransactionStatus::Mined, None)
2041 .await
2042 .unwrap();
2043
2044 assert_eq!(updated_tx.status, TransactionStatus::Mined);
2045 }
2046
2047 #[tokio::test]
2048 async fn test_updates_with_status_reason() {
2049 let mut mocks = default_test_mocks();
2050 let relayer = create_test_relayer();
2051 let tx = make_test_transaction(TransactionStatus::Submitted);
2052
2053 mocks
2054 .tx_repo
2055 .expect_partial_update()
2056 .withf(|_, update| {
2057 update.status == Some(TransactionStatus::Failed)
2058 && update.status_reason == Some("Transaction reverted on-chain".to_string())
2059 })
2060 .returning(|_, update| {
2061 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2062 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2063 updated_tx.status_reason = update.status_reason.clone();
2064 Ok(updated_tx)
2065 });
2066
2067 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2068 let updated_tx = evm_transaction
2069 .update_transaction_status_if_needed(
2070 tx.clone(),
2071 TransactionStatus::Failed,
2072 Some("Transaction reverted on-chain".to_string()),
2073 )
2074 .await
2075 .unwrap();
2076
2077 assert_eq!(updated_tx.status, TransactionStatus::Failed);
2078 assert_eq!(
2079 updated_tx.status_reason.as_deref(),
2080 Some("Transaction reverted on-chain")
2081 );
2082 }
2083 }
2084
2085 mod handle_sent_state_tests {
2087 use super::*;
2088
2089 #[tokio::test]
2090 async fn test_sent_state_recent_no_resend() {
2091 let mut mocks = default_test_mocks();
2092 let relayer = create_test_relayer();
2093
2094 let mut tx = make_test_transaction(TransactionStatus::Sent);
2095 tx.sent_at = Some((Utc::now() - Duration::seconds(10)).to_rfc3339());
2097
2098 mocks
2100 .network_repo
2101 .expect_get_by_chain_id()
2102 .returning(|_, _| Ok(Some(create_test_network_model())));
2103
2104 mocks.provider.expect_get_block_by_number().returning(|| {
2106 Box::pin(async {
2107 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2108 let mut block: Block = Block::default();
2109 block.header.gas_limit = 30_000_000u64;
2110 Ok(AnyRpcBlock::from(block))
2111 })
2112 });
2113
2114 mocks
2116 .job_producer
2117 .expect_produce_check_transaction_status_job()
2118 .returning(|_, _| Box::pin(async { Ok(()) }));
2119
2120 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2121 let result = evm_transaction.handle_sent_state(tx.clone()).await.unwrap();
2122
2123 assert_eq!(result.status, TransactionStatus::Sent);
2124 }
2125
2126 #[tokio::test]
2127 async fn test_sent_state_stuck_schedules_resubmit() {
2128 let mut mocks = default_test_mocks();
2129 let relayer = create_test_relayer();
2130
2131 let mut tx = make_test_transaction(TransactionStatus::Sent);
2132 tx.sent_at = Some((Utc::now() - Duration::seconds(60)).to_rfc3339());
2134
2135 mocks
2137 .network_repo
2138 .expect_get_by_chain_id()
2139 .returning(|_, _| Ok(Some(create_test_network_model())));
2140
2141 mocks.provider.expect_get_block_by_number().returning(|| {
2143 Box::pin(async {
2144 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2145 let mut block: Block = Block::default();
2146 block.header.gas_limit = 30_000_000u64;
2147 Ok(AnyRpcBlock::from(block))
2148 })
2149 });
2150
2151 mocks
2153 .job_producer
2154 .expect_produce_submit_transaction_job()
2155 .returning(|_, _| Box::pin(async { Ok(()) }));
2156
2157 mocks
2159 .job_producer
2160 .expect_produce_check_transaction_status_job()
2161 .returning(|_, _| Box::pin(async { Ok(()) }));
2162
2163 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2164 let result = evm_transaction.handle_sent_state(tx.clone()).await.unwrap();
2165
2166 assert_eq!(result.status, TransactionStatus::Sent);
2167 }
2168 }
2169
2170 mod prepare_noop_update_request_tests {
2172 use super::*;
2173
2174 #[tokio::test]
2175 async fn test_noop_request_without_cancellation() {
2176 let mocks = default_test_mocks_with_network();
2178 let relayer = create_test_relayer();
2179 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2180 tx.noop_count = Some(2);
2181 tx.is_canceled = Some(false);
2182
2183 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2184 let update_req = evm_transaction
2185 .prepare_noop_update_request(&tx, false, None)
2186 .await
2187 .unwrap();
2188
2189 assert_eq!(update_req.noop_count, Some(3));
2191 assert_eq!(update_req.is_canceled, Some(false));
2193 }
2194
2195 #[tokio::test]
2196 async fn test_noop_request_with_cancellation() {
2197 let mocks = default_test_mocks_with_network();
2199 let relayer = create_test_relayer();
2200 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2201 tx.noop_count = None;
2202 tx.is_canceled = Some(false);
2203
2204 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2205 let update_req = evm_transaction
2206 .prepare_noop_update_request(&tx, true, None)
2207 .await
2208 .unwrap();
2209
2210 assert_eq!(update_req.noop_count, Some(1));
2212 assert_eq!(update_req.is_canceled, Some(true));
2214 }
2215 }
2216
2217 mod handle_submitted_state_tests {
2219 use super::*;
2220
2221 #[tokio::test]
2222 async fn test_schedules_resubmit_job() {
2223 let mut mocks = default_test_mocks();
2224 let relayer = create_test_relayer();
2225
2226 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2228 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2229
2230 mocks
2232 .network_repo
2233 .expect_get_by_chain_id()
2234 .returning(|_, _| Ok(Some(create_test_network_model())));
2235
2236 mocks.provider.expect_get_block_by_number().returning(|| {
2238 Box::pin(async {
2239 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2240 let mut block: Block = Block::default();
2241 block.header.gas_limit = 30_000_000u64;
2242 Ok(AnyRpcBlock::from(block))
2243 })
2244 });
2245
2246 mocks
2248 .provider
2249 .expect_get_transaction_count()
2250 .returning(|_| Box::pin(async { Ok(10) }));
2251
2252 mocks
2254 .job_producer
2255 .expect_produce_submit_transaction_job()
2256 .returning(|_, _| Box::pin(async { Ok(()) }));
2257
2258 mocks
2260 .job_producer
2261 .expect_produce_check_transaction_status_job()
2262 .returning(|_, _| Box::pin(async { Ok(()) }));
2263
2264 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2265 let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2266
2267 assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2269 }
2270
2271 #[tokio::test]
2274 async fn test_nonce_gap_detected_schedules_health_skips_resubmit() {
2275 let mut mocks = default_test_mocks();
2276 let relayer = create_test_relayer();
2277
2278 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2279 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2280 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
2281 nonce: Some(274),
2282 hash: Some("0xhash".to_string()),
2283 raw: Some(vec![1, 2, 3]),
2284 ..tx.network_data.get_evm_transaction_data().unwrap()
2285 });
2286
2287 mocks
2289 .network_repo
2290 .expect_get_by_chain_id()
2291 .returning(|_, _| Ok(Some(create_test_network_model())));
2292
2293 mocks
2295 .provider
2296 .expect_get_transaction_count()
2297 .returning(|_| Box::pin(async { Ok(269) }));
2298
2299 mocks
2301 .tx_repo
2302 .expect_get_nonce_occupancy()
2303 .withf(|relayer_id, from, to| {
2304 relayer_id == "test-relayer-id" && *from == 269 && *to == 274
2305 })
2306 .returning(|_, from, to| Ok((from..to).map(|n| (n, None)).collect()));
2307
2308 mocks
2310 .job_producer
2311 .expect_produce_relayer_health_check_job()
2312 .withf(|job, _| {
2313 job.metadata.as_ref().map_or(false, |m| {
2314 m.get("health_check_action") == Some(&"nonce_health".to_string())
2315 })
2316 })
2317 .returning(|_, _| Box::pin(async { Ok(()) }));
2318
2319 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2323 let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2324
2325 assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2326 }
2327
2328 #[tokio::test]
2331 async fn test_nonce_gap_check_rpc_failure_proceeds_to_resubmit() {
2332 let mut mocks = default_test_mocks();
2333 let relayer = create_test_relayer();
2334
2335 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2336 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2337
2338 mocks
2339 .network_repo
2340 .expect_get_by_chain_id()
2341 .returning(|_, _| Ok(Some(create_test_network_model())));
2342
2343 mocks.provider.expect_get_block_by_number().returning(|| {
2344 Box::pin(async {
2345 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2346 let mut block: Block = Block::default();
2347 block.header.gas_limit = 30_000_000u64;
2348 Ok(AnyRpcBlock::from(block))
2349 })
2350 });
2351
2352 mocks
2354 .provider
2355 .expect_get_transaction_count()
2356 .returning(|_| {
2357 Box::pin(async {
2358 Err(crate::services::provider::ProviderError::Other(
2359 "rpc timeout".to_string(),
2360 ))
2361 })
2362 });
2363
2364 mocks
2366 .job_producer
2367 .expect_produce_submit_transaction_job()
2368 .returning(|_, _| Box::pin(async { Ok(()) }));
2369
2370 mocks
2371 .job_producer
2372 .expect_produce_check_transaction_status_job()
2373 .returning(|_, _| Box::pin(async { Ok(()) }));
2374
2375 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2376 let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2377
2378 assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2379 }
2380
2381 #[tokio::test]
2384 async fn test_no_gap_when_slots_filled_proceeds_to_resubmit() {
2385 let mut mocks = default_test_mocks();
2386 let relayer = create_test_relayer();
2387
2388 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2389 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2390 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
2391 nonce: Some(270),
2392 hash: Some("0xhash".to_string()),
2393 raw: Some(vec![1, 2, 3]),
2394 ..tx.network_data.get_evm_transaction_data().unwrap()
2395 });
2396
2397 mocks
2398 .network_repo
2399 .expect_get_by_chain_id()
2400 .returning(|_, _| Ok(Some(create_test_network_model())));
2401
2402 mocks.provider.expect_get_block_by_number().returning(|| {
2403 Box::pin(async {
2404 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2405 let mut block: Block = Block::default();
2406 block.header.gas_limit = 30_000_000u64;
2407 Ok(AnyRpcBlock::from(block))
2408 })
2409 });
2410
2411 mocks
2413 .provider
2414 .expect_get_transaction_count()
2415 .returning(|_| Box::pin(async { Ok(269) }));
2416
2417 mocks
2419 .tx_repo
2420 .expect_get_nonce_occupancy()
2421 .returning(|_, from, to| {
2422 Ok((from..to)
2423 .map(|n| (n, Some(TransactionStatus::Submitted)))
2424 .collect())
2425 });
2426
2427 mocks
2429 .job_producer
2430 .expect_produce_submit_transaction_job()
2431 .returning(|_, _| Box::pin(async { Ok(()) }));
2432
2433 mocks
2434 .job_producer
2435 .expect_produce_check_transaction_status_job()
2436 .returning(|_, _| Box::pin(async { Ok(()) }));
2437
2438 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2439 let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2440
2441 assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2442 }
2443 }
2444
2445 mod handle_pending_state_tests {
2447 use super::*;
2448
2449 #[tokio::test]
2450 async fn test_pending_state_no_noop() {
2451 let mut mocks = default_test_mocks();
2453 let relayer = create_test_relayer();
2454 let mut tx = make_test_transaction(TransactionStatus::Pending);
2455 tx.created_at = Utc::now().to_rfc3339(); mocks
2459 .network_repo
2460 .expect_get_by_chain_id()
2461 .returning(|_, _| Ok(Some(create_test_network_model())));
2462
2463 mocks.provider.expect_get_block_by_number().returning(|| {
2465 Box::pin(async {
2466 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2467 let mut block: Block = Block::default();
2468 block.header.gas_limit = 30_000_000u64;
2469 Ok(AnyRpcBlock::from(block))
2470 })
2471 });
2472
2473 mocks
2475 .job_producer
2476 .expect_produce_check_transaction_status_job()
2477 .returning(|_, _| Box::pin(async { Ok(()) }));
2478
2479 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2480 let result = evm_transaction
2481 .handle_pending_state(tx.clone())
2482 .await
2483 .unwrap();
2484
2485 assert_eq!(result.id, tx.id);
2487 assert_eq!(result.status, tx.status);
2488 assert_eq!(result.noop_count, tx.noop_count);
2489 }
2490
2491 #[tokio::test]
2492 async fn test_pending_state_with_noop() {
2493 let mut mocks = default_test_mocks();
2495 let relayer = create_test_relayer();
2496 let mut tx = make_test_transaction(TransactionStatus::Pending);
2497 tx.created_at = (Utc::now() - Duration::minutes(2)).to_rfc3339();
2498
2499 mocks
2501 .network_repo
2502 .expect_get_by_chain_id()
2503 .returning(|_, _| Ok(Some(create_test_network_model())));
2504
2505 mocks.provider.expect_get_block_by_number().returning(|| {
2507 Box::pin(async {
2508 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2509 let mut block: Block = Block::default();
2510 block.header.gas_limit = 30_000_000u64;
2511 Ok(AnyRpcBlock::from(block))
2512 })
2513 });
2514
2515 let tx_clone = tx.clone();
2518 mocks
2519 .tx_repo
2520 .expect_partial_update()
2521 .withf(move |id, update| {
2522 id == "test-tx-id"
2523 && update.status == Some(TransactionStatus::Failed)
2524 && update.status_reason.is_some()
2525 })
2526 .returning(move |_, update| {
2527 let mut updated_tx = tx_clone.clone();
2528 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2529 updated_tx.status_reason = update.status_reason.clone();
2530 Ok(updated_tx)
2531 });
2532 mocks
2534 .job_producer
2535 .expect_produce_send_notification_job()
2536 .returning(|_, _| Box::pin(async { Ok(()) }));
2537
2538 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2539 let result = evm_transaction
2540 .handle_pending_state(tx.clone())
2541 .await
2542 .unwrap();
2543
2544 assert_eq!(result.status, TransactionStatus::Failed);
2546 assert!(result.status_reason.is_some());
2547 assert!(result.status_reason.unwrap().contains("Pending state"));
2548 }
2549 }
2550
2551 mod handle_mined_state_tests {
2553 use super::*;
2554
2555 #[tokio::test]
2556 async fn test_updates_status_and_schedules_check() {
2557 let mut mocks = default_test_mocks();
2558 let relayer = create_test_relayer();
2559 let tx = make_test_transaction(TransactionStatus::Submitted);
2561
2562 mocks
2564 .job_producer
2565 .expect_produce_check_transaction_status_job()
2566 .returning(|_, _| Box::pin(async { Ok(()) }));
2567 mocks
2569 .tx_repo
2570 .expect_partial_update()
2571 .returning(|_, update| {
2572 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2573 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2574 Ok(updated_tx)
2575 });
2576
2577 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2578 let result = evm_transaction
2579 .handle_mined_state(tx.clone())
2580 .await
2581 .unwrap();
2582 assert_eq!(result.status, TransactionStatus::Mined);
2583 }
2584 }
2585
2586 mod handle_final_state_tests {
2588 use super::*;
2589
2590 #[tokio::test]
2591 async fn test_final_state_confirmed() {
2592 let mut mocks = default_test_mocks();
2593 let relayer = create_test_relayer();
2594 let tx = make_test_transaction(TransactionStatus::Submitted);
2595
2596 mocks
2598 .tx_repo
2599 .expect_partial_update()
2600 .returning(|_, update| {
2601 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2602 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2603 Ok(updated_tx)
2604 });
2605
2606 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2607 let result = evm_transaction
2608 .handle_final_state(tx.clone(), TransactionStatus::Confirmed, None)
2609 .await
2610 .unwrap();
2611 assert_eq!(result.status, TransactionStatus::Confirmed);
2612 }
2613
2614 #[tokio::test]
2615 async fn test_final_state_failed() {
2616 let mut mocks = default_test_mocks();
2617 let relayer = create_test_relayer();
2618 let tx = make_test_transaction(TransactionStatus::Submitted);
2619
2620 mocks
2622 .tx_repo
2623 .expect_partial_update()
2624 .returning(|_, update| {
2625 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2626 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2627 updated_tx.status_reason = update.status_reason.clone();
2628 Ok(updated_tx)
2629 });
2630
2631 let reason = "Transaction reverted on-chain (receipt status: failed)".to_string();
2632 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2633 let result = evm_transaction
2634 .handle_final_state(tx.clone(), TransactionStatus::Failed, Some(reason.clone()))
2635 .await
2636 .unwrap();
2637 assert_eq!(result.status, TransactionStatus::Failed);
2638 assert_eq!(result.status_reason.as_deref(), Some(reason.as_str()));
2639 }
2640
2641 #[tokio::test]
2642 async fn test_final_state_expired() {
2643 let mut mocks = default_test_mocks();
2644 let relayer = create_test_relayer();
2645 let tx = make_test_transaction(TransactionStatus::Submitted);
2646
2647 mocks
2649 .tx_repo
2650 .expect_partial_update()
2651 .returning(|_, update| {
2652 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2653 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2654 Ok(updated_tx)
2655 });
2656
2657 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2658 let result = evm_transaction
2659 .handle_final_state(tx.clone(), TransactionStatus::Expired, None)
2660 .await
2661 .unwrap();
2662 assert_eq!(result.status, TransactionStatus::Expired);
2663 }
2664 }
2665
2666 mod handle_status_impl_tests {
2668 use super::*;
2669
2670 #[tokio::test]
2671 async fn test_impl_submitted_branch() {
2672 let mut mocks = default_test_mocks();
2673 let relayer = create_test_relayer();
2674 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2675 tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
2676 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2678 evm_data.hash = Some("0xFakeHash".to_string());
2679 }
2680 mocks
2682 .provider
2683 .expect_get_transaction_receipt()
2684 .returning(|_| Box::pin(async { Ok(None) }));
2685 mocks
2687 .network_repo
2688 .expect_get_by_chain_id()
2689 .returning(|_, _| Ok(Some(create_test_network_model())));
2690 mocks
2692 .job_producer
2693 .expect_produce_check_transaction_status_job()
2694 .returning(|_, _| Box::pin(async { Ok(()) }));
2695 mocks
2697 .tx_repo
2698 .expect_partial_update()
2699 .returning(|_, update| {
2700 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2701 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2702 Ok(updated_tx)
2703 });
2704
2705 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2706 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2707 assert_eq!(result.status, TransactionStatus::Submitted);
2708 }
2709
2710 #[tokio::test]
2711 async fn test_impl_mined_branch() {
2712 let mut mocks = default_test_mocks();
2713 let relayer = create_test_relayer();
2714 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2715 tx.created_at = (Utc::now() - Duration::minutes(1)).to_rfc3339();
2717 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2719 evm_data.hash = Some("0xFakeHash".to_string());
2720 }
2721 mocks
2723 .provider
2724 .expect_get_transaction_receipt()
2725 .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) }));
2726 mocks
2728 .provider
2729 .expect_get_block_number()
2730 .return_once(|| Box::pin(async { Ok(100) }));
2731 mocks
2733 .network_repo
2734 .expect_get_by_chain_id()
2735 .returning(|_, _| Ok(Some(create_test_network_model())));
2736 mocks
2738 .job_producer
2739 .expect_produce_send_notification_job()
2740 .returning(|_, _| Box::pin(async { Ok(()) }));
2741 mocks.tx_repo.expect_get_by_id().returning(|_| {
2743 let updated_tx = make_test_transaction(TransactionStatus::Mined);
2744 Ok(updated_tx)
2745 });
2746 mocks
2748 .tx_repo
2749 .expect_partial_update()
2750 .returning(|_, update| {
2751 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2752 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2753 Ok(updated_tx)
2754 });
2755
2756 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2757 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2758 assert_eq!(result.status, TransactionStatus::Mined);
2759 }
2760
2761 #[tokio::test]
2762 async fn test_impl_final_confirmed_branch() {
2763 let mut mocks = default_test_mocks();
2764 let relayer = create_test_relayer();
2765 let tx = make_test_transaction(TransactionStatus::Confirmed);
2767
2768 mocks
2771 .tx_repo
2772 .expect_partial_update()
2773 .returning(|_, update| {
2774 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2775 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2776 Ok(updated_tx)
2777 });
2778
2779 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2780 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2781 assert_eq!(result.status, TransactionStatus::Confirmed);
2782 }
2783
2784 #[tokio::test]
2785 async fn test_impl_final_failed_branch() {
2786 let mut mocks = default_test_mocks();
2787 let relayer = create_test_relayer();
2788 let tx = make_test_transaction(TransactionStatus::Failed);
2790
2791 mocks
2792 .tx_repo
2793 .expect_partial_update()
2794 .returning(|_, update| {
2795 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2796 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2797 Ok(updated_tx)
2798 });
2799
2800 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2801 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2802 assert_eq!(result.status, TransactionStatus::Failed);
2803 }
2804
2805 #[tokio::test]
2808 async fn test_impl_submitted_to_failed_sets_status_reason() {
2809 let mut mocks = default_test_mocks();
2810 let relayer = create_test_relayer();
2811 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2812 tx.created_at = (Utc::now() - Duration::minutes(1)).to_rfc3339();
2813 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2814 evm_data.hash = Some("0xFakeHash".to_string());
2815 }
2816
2817 mocks
2819 .provider
2820 .expect_get_transaction_receipt()
2821 .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(false, Some(100)))) }));
2822
2823 let tx_clone = tx.clone();
2825 mocks.tx_repo.expect_get_by_id().returning(move |_| {
2826 let mut reloaded = tx_clone.clone();
2827 reloaded.status = TransactionStatus::Submitted;
2828 Ok(reloaded)
2829 });
2830
2831 mocks
2833 .tx_repo
2834 .expect_partial_update()
2835 .withf(|_, update| {
2836 update.status == Some(TransactionStatus::Failed)
2837 && update.status_reason.is_some()
2838 })
2839 .returning(|_, update| {
2840 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2841 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2842 updated_tx.status_reason = update.status_reason.clone();
2843 Ok(updated_tx)
2844 });
2845
2846 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2847 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2848 assert_eq!(result.status, TransactionStatus::Failed);
2849 assert!(result.status_reason.is_some());
2850 assert!(
2851 result
2852 .status_reason
2853 .as_ref()
2854 .unwrap()
2855 .contains("reverted on-chain"),
2856 "Expected on-chain revert reason, got: {:?}",
2857 result.status_reason
2858 );
2859 }
2860
2861 #[tokio::test]
2862 async fn test_impl_final_expired_branch() {
2863 let mut mocks = default_test_mocks();
2864 let relayer = create_test_relayer();
2865 let tx = make_test_transaction(TransactionStatus::Expired);
2867
2868 mocks
2869 .tx_repo
2870 .expect_partial_update()
2871 .returning(|_, update| {
2872 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2873 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2874 Ok(updated_tx)
2875 });
2876
2877 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2878 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2879 assert_eq!(result.status, TransactionStatus::Expired);
2880 }
2881 }
2882
2883 mod circuit_breaker_tests {
2885 use super::*;
2886 use crate::jobs::StatusCheckContext;
2887
2888 fn create_triggered_context() -> StatusCheckContext {
2890 StatusCheckContext::new(
2891 30, 50, 60, 25, 75, NetworkType::Evm,
2897 )
2898 }
2899
2900 fn create_safe_context() -> StatusCheckContext {
2902 StatusCheckContext::new(
2903 5, 10, 15, 25, 75, NetworkType::Evm,
2909 )
2910 }
2911
2912 fn create_total_triggered_context() -> StatusCheckContext {
2914 StatusCheckContext::new(
2915 5, 80, 100, 25, 75, NetworkType::Evm,
2921 )
2922 }
2923
2924 #[tokio::test]
2925 async fn test_circuit_breaker_pending_marks_as_failed() {
2926 let mut mocks = default_test_mocks();
2927 let relayer = create_test_relayer();
2928 let tx = make_test_transaction(TransactionStatus::Pending);
2929
2930 mocks
2932 .tx_repo
2933 .expect_partial_update()
2934 .withf(|_, update| update.status == Some(TransactionStatus::Failed))
2935 .returning(|_, update| {
2936 let mut updated_tx = make_test_transaction(TransactionStatus::Pending);
2937 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2938 updated_tx.status_reason = update.status_reason.clone();
2939 Ok(updated_tx)
2940 });
2941
2942 mocks
2944 .job_producer
2945 .expect_produce_send_notification_job()
2946 .returning(|_, _| Box::pin(async { Ok(()) }));
2947
2948 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2949 let ctx = create_triggered_context();
2950
2951 let result = evm_transaction
2952 .handle_status_impl(tx, Some(ctx))
2953 .await
2954 .unwrap();
2955
2956 assert_eq!(result.status, TransactionStatus::Failed);
2957 assert!(result.status_reason.is_some());
2958 assert!(result.status_reason.unwrap().contains("consecutive errors"));
2959 }
2960
2961 #[tokio::test]
2962 async fn test_circuit_breaker_sent_marks_as_failed() {
2963 let mut mocks = default_test_mocks();
2964 let relayer = create_test_relayer();
2965 let tx = make_test_transaction(TransactionStatus::Sent);
2966
2967 mocks
2969 .tx_repo
2970 .expect_partial_update()
2971 .withf(|_, update| update.status == Some(TransactionStatus::Failed))
2972 .returning(|_, update| {
2973 let mut updated_tx = make_test_transaction(TransactionStatus::Sent);
2974 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2975 updated_tx.status_reason = update.status_reason.clone();
2976 Ok(updated_tx)
2977 });
2978
2979 mocks
2981 .job_producer
2982 .expect_produce_send_notification_job()
2983 .returning(|_, _| Box::pin(async { Ok(()) }));
2984
2985 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2986 let ctx = create_triggered_context();
2987
2988 let result = evm_transaction
2989 .handle_status_impl(tx, Some(ctx))
2990 .await
2991 .unwrap();
2992
2993 assert_eq!(result.status, TransactionStatus::Failed);
2994 }
2995
2996 #[tokio::test]
2997 async fn test_circuit_breaker_submitted_triggers_noop() {
2998 let mut mocks = default_test_mocks();
2999 let relayer = create_test_relayer();
3000 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3001 tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3002
3003 mocks
3005 .network_repo
3006 .expect_get_by_chain_id()
3007 .returning(|_, _| Ok(Some(create_test_network_model())));
3008
3009 mocks
3011 .tx_repo
3012 .expect_partial_update()
3013 .returning(|_, update| {
3014 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3015 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3016 updated_tx.status_reason = update.status_reason.clone();
3017 updated_tx.noop_count = update.noop_count;
3018 Ok(updated_tx)
3019 });
3020
3021 mocks
3023 .job_producer
3024 .expect_produce_submit_transaction_job()
3025 .returning(|_, _| Box::pin(async { Ok(()) }));
3026
3027 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3028 let ctx = create_triggered_context();
3029
3030 let result = evm_transaction
3031 .handle_status_impl(tx, Some(ctx))
3032 .await
3033 .unwrap();
3034
3035 assert!(result.noop_count.is_some());
3037 }
3038
3039 #[tokio::test]
3040 async fn test_circuit_breaker_noop_tx_excluded() {
3041 let mut mocks = default_test_mocks();
3042 let relayer = create_test_relayer();
3043
3044 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3046 tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3047 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3048 evm_data.to = Some(evm_data.from.clone()); evm_data.value = U256::from(0);
3050 evm_data.data = Some("0x".to_string());
3051 evm_data.hash = Some("0xFakeHash".to_string());
3052 }
3053
3054 mocks
3057 .provider
3058 .expect_get_transaction_receipt()
3059 .returning(|_| Box::pin(async { Ok(None) }));
3060
3061 mocks
3062 .network_repo
3063 .expect_get_by_chain_id()
3064 .returning(|_, _| Ok(Some(create_test_network_model())));
3065
3066 mocks
3067 .job_producer
3068 .expect_produce_check_transaction_status_job()
3069 .returning(|_, _| Box::pin(async { Ok(()) }));
3070
3071 mocks
3073 .job_producer
3074 .expect_produce_submit_transaction_job()
3075 .returning(|_, _| Box::pin(async { Ok(()) }));
3076
3077 mocks
3078 .tx_repo
3079 .expect_partial_update()
3080 .returning(|_, update| {
3081 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3082 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3083 Ok(updated_tx)
3084 });
3085
3086 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3087 let ctx = create_triggered_context();
3088
3089 let result = evm_transaction
3090 .handle_status_impl(tx, Some(ctx))
3091 .await
3092 .unwrap();
3093
3094 assert_eq!(result.status, TransactionStatus::Submitted);
3096 }
3097
3098 #[tokio::test]
3099 async fn test_circuit_breaker_total_failures_triggers() {
3100 let mut mocks = default_test_mocks();
3101 let relayer = create_test_relayer();
3102 let tx = make_test_transaction(TransactionStatus::Pending);
3103
3104 mocks
3106 .tx_repo
3107 .expect_partial_update()
3108 .withf(|_, update| update.status == Some(TransactionStatus::Failed))
3109 .returning(|_, update| {
3110 let mut updated_tx = make_test_transaction(TransactionStatus::Pending);
3111 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3112 updated_tx.status_reason = update.status_reason.clone();
3113 Ok(updated_tx)
3114 });
3115
3116 mocks
3117 .job_producer
3118 .expect_produce_send_notification_job()
3119 .returning(|_, _| Box::pin(async { Ok(()) }));
3120
3121 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3122 let ctx = create_total_triggered_context();
3124
3125 let result = evm_transaction
3126 .handle_status_impl(tx, Some(ctx))
3127 .await
3128 .unwrap();
3129
3130 assert_eq!(result.status, TransactionStatus::Failed);
3131 assert!(result.status_reason.is_some());
3132 }
3133
3134 #[tokio::test]
3135 async fn test_circuit_breaker_below_threshold_continues_normally() {
3136 let mut mocks = default_test_mocks();
3137 let relayer = create_test_relayer();
3138 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3139 tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3140
3141 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3142 evm_data.hash = Some("0xFakeHash".to_string());
3143 }
3144
3145 mocks
3147 .provider
3148 .expect_get_transaction_receipt()
3149 .returning(|_| Box::pin(async { Ok(None) }));
3150
3151 mocks
3152 .network_repo
3153 .expect_get_by_chain_id()
3154 .returning(|_, _| Ok(Some(create_test_network_model())));
3155
3156 mocks
3157 .job_producer
3158 .expect_produce_check_transaction_status_job()
3159 .returning(|_, _| Box::pin(async { Ok(()) }));
3160
3161 mocks
3162 .tx_repo
3163 .expect_partial_update()
3164 .returning(|_, update| {
3165 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3166 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3167 Ok(updated_tx)
3168 });
3169
3170 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3171 let ctx = create_safe_context();
3172
3173 let result = evm_transaction
3174 .handle_status_impl(tx, Some(ctx))
3175 .await
3176 .unwrap();
3177
3178 assert_eq!(result.status, TransactionStatus::Submitted);
3180 }
3181
3182 #[tokio::test]
3183 async fn test_circuit_breaker_no_context_continues_normally() {
3184 let mut mocks = default_test_mocks();
3185 let relayer = create_test_relayer();
3186 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3187 tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3188
3189 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3190 evm_data.hash = Some("0xFakeHash".to_string());
3191 }
3192
3193 mocks
3195 .provider
3196 .expect_get_transaction_receipt()
3197 .returning(|_| Box::pin(async { Ok(None) }));
3198
3199 mocks
3200 .network_repo
3201 .expect_get_by_chain_id()
3202 .returning(|_, _| Ok(Some(create_test_network_model())));
3203
3204 mocks
3205 .job_producer
3206 .expect_produce_check_transaction_status_job()
3207 .returning(|_, _| Box::pin(async { Ok(()) }));
3208
3209 mocks
3210 .tx_repo
3211 .expect_partial_update()
3212 .returning(|_, update| {
3213 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3214 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3215 Ok(updated_tx)
3216 });
3217
3218 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3219
3220 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3222
3223 assert_eq!(result.status, TransactionStatus::Submitted);
3225 }
3226
3227 #[tokio::test]
3228 async fn test_circuit_breaker_final_state_early_return() {
3229 let mocks = default_test_mocks();
3230 let relayer = create_test_relayer();
3231 let tx = make_test_transaction(TransactionStatus::Confirmed);
3233
3234 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3235 let ctx = create_triggered_context();
3236
3237 let result = evm_transaction
3239 .handle_status_impl(tx, Some(ctx))
3240 .await
3241 .unwrap();
3242
3243 assert_eq!(result.status, TransactionStatus::Confirmed);
3244 }
3245 }
3246
3247 mod hash_recovery_tests {
3249 use super::*;
3250
3251 #[tokio::test]
3252 async fn test_should_try_hash_recovery_not_submitted() {
3253 let mocks = default_test_mocks();
3254 let relayer = create_test_relayer();
3255
3256 let mut tx = make_test_transaction(TransactionStatus::Sent);
3257 tx.hashes = vec![
3258 "0xHash1".to_string(),
3259 "0xHash2".to_string(),
3260 "0xHash3".to_string(),
3261 ];
3262
3263 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3264 let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3265
3266 assert!(
3267 !result,
3268 "Should not attempt recovery for non-Submitted transactions"
3269 );
3270 }
3271
3272 #[tokio::test]
3273 async fn test_should_try_hash_recovery_not_enough_hashes() {
3274 let mocks = default_test_mocks();
3275 let relayer = create_test_relayer();
3276
3277 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3278 tx.hashes = vec!["0xHash1".to_string()]; tx.sent_at = Some((Utc::now() - Duration::minutes(3)).to_rfc3339());
3280
3281 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3282 let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3283
3284 assert!(
3285 !result,
3286 "Should not attempt recovery with insufficient hashes"
3287 );
3288 }
3289
3290 #[tokio::test]
3291 async fn test_should_try_hash_recovery_too_recent() {
3292 let mocks = default_test_mocks();
3293 let relayer = create_test_relayer();
3294
3295 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3296 tx.hashes = vec![
3297 "0xHash1".to_string(),
3298 "0xHash2".to_string(),
3299 "0xHash3".to_string(),
3300 ];
3301 tx.sent_at = Some(Utc::now().to_rfc3339()); let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3304 let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3305
3306 assert!(
3307 !result,
3308 "Should not attempt recovery for recently sent transactions"
3309 );
3310 }
3311
3312 #[tokio::test]
3313 async fn test_should_try_hash_recovery_success() {
3314 let mocks = default_test_mocks();
3315 let relayer = create_test_relayer();
3316
3317 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3318 tx.hashes = vec![
3319 "0xHash1".to_string(),
3320 "0xHash2".to_string(),
3321 "0xHash3".to_string(),
3322 ];
3323 tx.sent_at = Some((Utc::now() - Duration::minutes(3)).to_rfc3339());
3324
3325 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3326 let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3327
3328 assert!(
3329 result,
3330 "Should attempt recovery for stuck transactions with multiple hashes"
3331 );
3332 }
3333
3334 #[tokio::test]
3335 async fn test_try_recover_no_historical_hash_found() {
3336 let mut mocks = default_test_mocks();
3337 let relayer = create_test_relayer();
3338
3339 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3340 tx.hashes = vec![
3341 "0xHash1".to_string(),
3342 "0xHash2".to_string(),
3343 "0xHash3".to_string(),
3344 ];
3345
3346 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3347 evm_data.hash = Some("0xHash3".to_string());
3348 }
3349
3350 mocks
3352 .provider
3353 .expect_get_transaction_receipt()
3354 .returning(|_| Box::pin(async { Ok(None) }));
3355
3356 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3357 let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
3358 let result = evm_transaction
3359 .try_recover_with_historical_hashes(&tx, &evm_data)
3360 .await
3361 .unwrap();
3362
3363 assert!(
3364 result.is_none(),
3365 "Should return None when no historical hash is found"
3366 );
3367 }
3368
3369 #[tokio::test]
3370 async fn test_try_recover_finds_mined_historical_hash() {
3371 let mut mocks = default_test_mocks();
3372 let relayer = create_test_relayer();
3373
3374 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3375 tx.hashes = vec![
3376 "0xHash1".to_string(),
3377 "0xHash2".to_string(), "0xHash3".to_string(),
3379 ];
3380
3381 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3382 evm_data.hash = Some("0xHash3".to_string()); }
3384
3385 mocks
3387 .provider
3388 .expect_get_transaction_receipt()
3389 .returning(|hash| {
3390 if hash == "0xHash2" {
3391 Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) })
3392 } else {
3393 Box::pin(async { Ok(None) })
3394 }
3395 });
3396
3397 let tx_clone = tx.clone();
3399 mocks
3400 .tx_repo
3401 .expect_partial_update()
3402 .returning(move |_, update| {
3403 let mut updated_tx = tx_clone.clone();
3404 if let Some(status) = update.status {
3405 updated_tx.status = status;
3406 }
3407 if let Some(NetworkTransactionData::Evm(ref evm_data)) = update.network_data {
3408 if let NetworkTransactionData::Evm(ref mut updated_evm) =
3409 updated_tx.network_data
3410 {
3411 updated_evm.hash = evm_data.hash.clone();
3412 }
3413 }
3414 Ok(updated_tx)
3415 });
3416
3417 mocks
3419 .job_producer
3420 .expect_produce_send_notification_job()
3421 .returning(|_, _| Box::pin(async { Ok(()) }));
3422
3423 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3424 let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
3425 let result = evm_transaction
3426 .try_recover_with_historical_hashes(&tx, &evm_data)
3427 .await
3428 .unwrap();
3429
3430 assert!(result.is_some(), "Should recover the transaction");
3431 let recovered_tx = result.unwrap();
3432 assert_eq!(recovered_tx.status, TransactionStatus::Mined);
3433 }
3434
3435 #[tokio::test]
3436 async fn test_try_recover_network_error_continues() {
3437 let mut mocks = default_test_mocks();
3438 let relayer = create_test_relayer();
3439
3440 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3441 tx.hashes = vec![
3442 "0xHash1".to_string(),
3443 "0xHash2".to_string(), "0xHash3".to_string(), ];
3446
3447 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3448 evm_data.hash = Some("0xHash1".to_string());
3449 }
3450
3451 mocks
3453 .provider
3454 .expect_get_transaction_receipt()
3455 .returning(|hash| {
3456 if hash == "0xHash2" {
3457 Box::pin(async { Err(crate::services::provider::ProviderError::Timeout) })
3458 } else if hash == "0xHash3" {
3459 Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) })
3460 } else {
3461 Box::pin(async { Ok(None) })
3462 }
3463 });
3464
3465 let tx_clone = tx.clone();
3467 mocks
3468 .tx_repo
3469 .expect_partial_update()
3470 .returning(move |_, update| {
3471 let mut updated_tx = tx_clone.clone();
3472 if let Some(status) = update.status {
3473 updated_tx.status = status;
3474 }
3475 Ok(updated_tx)
3476 });
3477
3478 mocks
3480 .job_producer
3481 .expect_produce_send_notification_job()
3482 .returning(|_, _| Box::pin(async { Ok(()) }));
3483
3484 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3485 let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
3486 let result = evm_transaction
3487 .try_recover_with_historical_hashes(&tx, &evm_data)
3488 .await
3489 .unwrap();
3490
3491 assert!(
3492 result.is_some(),
3493 "Should continue checking after network error and find mined hash"
3494 );
3495 }
3496
3497 #[tokio::test]
3498 async fn test_update_transaction_with_corrected_hash() {
3499 let mut mocks = default_test_mocks();
3500 let relayer = create_test_relayer();
3501
3502 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3503 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3504 evm_data.hash = Some("0xWrongHash".to_string());
3505 }
3506
3507 mocks
3509 .tx_repo
3510 .expect_partial_update()
3511 .returning(move |_, update| {
3512 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3513 if let Some(status) = update.status {
3514 updated_tx.status = status;
3515 }
3516 if let Some(NetworkTransactionData::Evm(ref evm_data)) = update.network_data {
3517 if let NetworkTransactionData::Evm(ref mut updated_evm) =
3518 updated_tx.network_data
3519 {
3520 updated_evm.hash = evm_data.hash.clone();
3521 }
3522 }
3523 Ok(updated_tx)
3524 });
3525
3526 mocks
3528 .job_producer
3529 .expect_produce_send_notification_job()
3530 .returning(|_, _| Box::pin(async { Ok(()) }));
3531
3532 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3533 let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
3534 let result = evm_transaction
3535 .update_transaction_with_corrected_hash(
3536 &tx,
3537 &evm_data,
3538 "0xCorrectHash",
3539 TransactionStatus::Mined,
3540 )
3541 .await
3542 .unwrap();
3543
3544 assert_eq!(result.status, TransactionStatus::Mined);
3545 if let NetworkTransactionData::Evm(ref updated_evm) = result.network_data {
3546 assert_eq!(updated_evm.hash.as_ref().unwrap(), "0xCorrectHash");
3547 }
3548 }
3549 }
3550
3551 mod check_transaction_status_edge_cases {
3553 use super::*;
3554
3555 #[tokio::test]
3556 async fn test_missing_hash_returns_error() {
3557 let mocks = default_test_mocks();
3558 let relayer = create_test_relayer();
3559
3560 let tx = make_test_transaction(TransactionStatus::Submitted);
3561 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3564 let result = evm_transaction.check_transaction_status(&tx).await;
3565
3566 assert!(result.is_err(), "Should return error when hash is missing");
3567 }
3568
3569 #[tokio::test]
3570 async fn test_pending_status_early_return() {
3571 let mocks = default_test_mocks();
3572 let relayer = create_test_relayer();
3573
3574 let tx = make_test_transaction(TransactionStatus::Pending);
3575
3576 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3577 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
3578
3579 assert_eq!(
3580 status,
3581 TransactionStatus::Pending,
3582 "Should return Pending without querying blockchain"
3583 );
3584 }
3585
3586 #[tokio::test]
3587 async fn test_sent_status_early_return() {
3588 let mocks = default_test_mocks();
3589 let relayer = create_test_relayer();
3590
3591 let tx = make_test_transaction(TransactionStatus::Sent);
3592
3593 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3594 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
3595
3596 assert_eq!(
3597 status,
3598 TransactionStatus::Sent,
3599 "Should return Sent without querying blockchain"
3600 );
3601 }
3602
3603 #[tokio::test]
3604 async fn test_final_state_early_return() {
3605 let mocks = default_test_mocks();
3606 let relayer = create_test_relayer();
3607
3608 let tx = make_test_transaction(TransactionStatus::Confirmed);
3609
3610 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3611 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
3612
3613 assert_eq!(
3614 status,
3615 TransactionStatus::Confirmed,
3616 "Should return final state without querying blockchain"
3617 );
3618 }
3619 }
3620
3621 mod nonce_recovery_tests {
3622 use super::*;
3623 use crate::domain::transaction::evm::evm_transaction::TX_NONCE_RECONCILE_TRIGGER;
3624 use crate::jobs::StatusCheckContext;
3625
3626 #[tokio::test]
3628 async fn test_nonce_recovery_nonce_consumed_externally() {
3629 let mut mocks = default_test_mocks();
3630 let relayer = create_test_relayer();
3631
3632 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3633 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
3634 nonce: Some(5),
3635 hash: Some("0xhash".to_string()),
3636 raw: Some(vec![1, 2, 3]),
3637 ..tx.network_data.get_evm_transaction_data().unwrap()
3638 });
3639 tx.sent_at = Some(Utc::now().to_rfc3339());
3640
3641 mocks
3643 .provider
3644 .expect_get_transaction_receipt()
3645 .returning(|_| Box::pin(async { Ok(None) }));
3646
3647 mocks
3649 .provider
3650 .expect_get_transaction_count()
3651 .returning(|_| Box::pin(async { Ok(10) }));
3652
3653 let tx_clone = tx.clone();
3655 mocks
3656 .tx_repo
3657 .expect_partial_update()
3658 .withf(|_, update| {
3659 update.status == Some(TransactionStatus::Failed)
3660 && update
3661 .status_reason
3662 .as_ref()
3663 .map(|r| r.contains("consumed externally"))
3664 .unwrap_or(false)
3665 })
3666 .returning(move |_, update| {
3667 let mut updated_tx = tx_clone.clone();
3668 updated_tx.status = update.status.unwrap();
3669 updated_tx.status_reason = update.status_reason.clone();
3670 Ok(updated_tx)
3671 });
3672
3673 mocks
3675 .job_producer
3676 .expect_produce_relayer_health_check_job()
3677 .withf(|job, scheduled_on| {
3678 job.relayer_id == "test-relayer-id"
3679 && job.metadata.as_ref().map_or(false, |m| {
3680 m.get("health_check_action") == Some(&"nonce_health".to_string())
3681 })
3682 && scheduled_on.is_none()
3683 })
3684 .returning(|_, _| Box::pin(async { Ok(()) }));
3685
3686 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3687 let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
3688
3689 assert!(result.is_ok());
3690 let recovered = result.unwrap();
3691 assert!(recovered.is_some(), "Expected Some(tx) for consumed nonce");
3692 assert_eq!(recovered.unwrap().status, TransactionStatus::Failed);
3693 }
3694
3695 #[tokio::test]
3697 async fn test_nonce_recovery_nonce_not_consumed() {
3698 let mut mocks = default_test_mocks();
3699 let relayer = create_test_relayer();
3700
3701 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3702 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
3703 nonce: Some(5),
3704 hash: Some("0xhash".to_string()),
3705 raw: Some(vec![1, 2, 3]),
3706 ..tx.network_data.get_evm_transaction_data().unwrap()
3707 });
3708
3709 mocks
3711 .provider
3712 .expect_get_transaction_receipt()
3713 .returning(|_| Box::pin(async { Ok(None) }));
3714
3715 mocks
3717 .provider
3718 .expect_get_transaction_count()
3719 .returning(|_| Box::pin(async { Ok(5) }));
3720
3721 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3722 let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
3723
3724 assert!(result.is_ok());
3725 assert!(
3726 result.unwrap().is_none(),
3727 "Expected None when nonce not consumed"
3728 );
3729 }
3730
3731 #[tokio::test]
3733 async fn test_nonce_recovery_receipt_found() {
3734 let mut mocks = default_test_mocks();
3735 let relayer = create_test_relayer();
3736
3737 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3738 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
3739 nonce: Some(5),
3740 hash: Some("0xhash".to_string()),
3741 raw: Some(vec![1, 2, 3]),
3742 ..tx.network_data.get_evm_transaction_data().unwrap()
3743 });
3744
3745 mocks
3747 .provider
3748 .expect_get_transaction_receipt()
3749 .returning(|_| {
3750 let receipt = make_mock_receipt(true, Some(100));
3751 Box::pin(async move { Ok(Some(receipt)) })
3752 });
3753
3754 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3755 let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
3756
3757 assert!(result.is_ok());
3758 assert!(
3759 result.unwrap().is_none(),
3760 "Expected None when receipt found — defer to normal flow"
3761 );
3762 }
3763
3764 #[tokio::test]
3767 async fn test_nonce_recovery_rpc_error_prevents_force_fail() {
3768 let mut mocks = default_test_mocks();
3769 let relayer = create_test_relayer();
3770
3771 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3772 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
3773 nonce: Some(5),
3774 hash: Some("0xhash".to_string()),
3775 raw: Some(vec![1, 2, 3]),
3776 ..tx.network_data.get_evm_transaction_data().unwrap()
3777 });
3778
3779 mocks
3781 .provider
3782 .expect_get_transaction_receipt()
3783 .returning(|_| {
3784 Box::pin(async {
3785 Err(crate::services::provider::ProviderError::Other(
3786 "RPC timeout".to_string(),
3787 ))
3788 })
3789 });
3790
3791 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3795 let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
3796
3797 assert!(result.is_ok());
3798 assert!(
3799 result.unwrap().is_none(),
3800 "Expected None when RPC errors occurred — must not force-fail on incomplete data"
3801 );
3802 }
3803
3804 #[tokio::test]
3806 async fn test_handle_status_impl_nonce_recovery_hint() {
3807 let mut mocks = default_test_mocks();
3808 let relayer = create_test_relayer();
3809
3810 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3811 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
3812 nonce: Some(5),
3813 hash: Some("0xhash".to_string()),
3814 raw: Some(vec![1, 2, 3]),
3815 ..tx.network_data.get_evm_transaction_data().unwrap()
3816 });
3817 tx.sent_at = Some(Utc::now().to_rfc3339());
3818
3819 mocks
3821 .provider
3822 .expect_get_transaction_receipt()
3823 .returning(|_| Box::pin(async { Ok(None) }));
3824
3825 mocks
3827 .provider
3828 .expect_get_transaction_count()
3829 .returning(|_| Box::pin(async { Ok(10) }));
3830
3831 let tx_clone = tx.clone();
3833 mocks
3834 .tx_repo
3835 .expect_partial_update()
3836 .returning(move |_, update| {
3837 let mut updated_tx = tx_clone.clone();
3838 if let Some(status) = update.status {
3839 updated_tx.status = status;
3840 }
3841 updated_tx.status_reason = update.status_reason.clone();
3842 Ok(updated_tx)
3843 });
3844
3845 mocks
3847 .job_producer
3848 .expect_produce_relayer_health_check_job()
3849 .returning(|_, _| Box::pin(async { Ok(()) }));
3850
3851 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3852
3853 let mut metadata = std::collections::HashMap::new();
3855 metadata.insert(
3856 TX_NONCE_RECONCILE_TRIGGER.to_string(),
3857 "NonceTooLow".to_string(),
3858 );
3859 let context = StatusCheckContext::default().with_job_metadata(Some(metadata));
3860
3861 let result = evm_transaction.handle_status_impl(tx, Some(context)).await;
3862 assert!(result.is_ok());
3863 assert_eq!(result.unwrap().status, TransactionStatus::Failed);
3864 }
3865 }
3866
3867 mod circuit_breaker_sent_state_tests {
3868 use super::*;
3869 use crate::jobs::StatusCheckContext;
3870
3871 #[tokio::test]
3873 async fn test_circuit_breaker_sent_with_nonce_issues_noop() {
3874 let mut mocks = default_test_mocks_with_network();
3875 let relayer = create_test_relayer();
3876
3877 let mut tx = make_test_transaction(TransactionStatus::Sent);
3878 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
3879 nonce: Some(5),
3880 hash: Some("0xhash".to_string()),
3881 raw: Some(vec![1, 2, 3]),
3882 ..tx.network_data.get_evm_transaction_data().unwrap()
3883 });
3884 tx.sent_at = Some(Utc::now().to_rfc3339());
3885
3886 let tx_clone = tx.clone();
3888 mocks
3889 .tx_repo
3890 .expect_partial_update()
3891 .returning(move |_, update| {
3892 let mut updated_tx = tx_clone.clone();
3893 if let Some(status) = update.status {
3894 updated_tx.status = status;
3895 }
3896 updated_tx.status_reason = update.status_reason.clone();
3897 Ok(updated_tx)
3898 });
3899
3900 mocks
3902 .job_producer
3903 .expect_produce_submit_transaction_job()
3904 .times(1)
3905 .returning(|_, _| Box::pin(async { Ok(()) }));
3906
3907 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3908
3909 let ctx = StatusCheckContext::new(100, 200, 250, 15, 45, NetworkType::Evm);
3911
3912 let result = evm_transaction
3913 .handle_circuit_breaker_safely(tx, &ctx)
3914 .await;
3915 assert!(result.is_ok());
3916 }
3917
3918 #[tokio::test]
3920 async fn test_circuit_breaker_sent_without_nonce_marks_failed() {
3921 let mut mocks = default_test_mocks();
3922 let relayer = create_test_relayer();
3923
3924 let mut tx = make_test_transaction(TransactionStatus::Sent);
3925 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
3927 nonce: None,
3928 hash: None,
3929 raw: None,
3930 ..tx.network_data.get_evm_transaction_data().unwrap()
3931 });
3932 tx.sent_at = Some(Utc::now().to_rfc3339());
3933
3934 let tx_clone = tx.clone();
3936 mocks
3937 .tx_repo
3938 .expect_partial_update()
3939 .withf(|_, update| update.status == Some(TransactionStatus::Failed))
3940 .returning(move |_, update| {
3941 let mut updated_tx = tx_clone.clone();
3942 updated_tx.status = update.status.unwrap();
3943 Ok(updated_tx)
3944 });
3945
3946 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3947
3948 let ctx = StatusCheckContext::new(100, 200, 250, 15, 45, NetworkType::Evm);
3949
3950 let result = evm_transaction
3951 .handle_circuit_breaker_safely(tx, &ctx)
3952 .await;
3953 assert!(result.is_ok());
3954 assert_eq!(result.unwrap().status, TransactionStatus::Failed);
3955 }
3956
3957 #[tokio::test]
3959 async fn test_circuit_breaker_pending_marks_failed() {
3960 let mut mocks = default_test_mocks();
3961 let relayer = create_test_relayer();
3962
3963 let tx = make_test_transaction(TransactionStatus::Pending);
3964
3965 let tx_clone = tx.clone();
3967 mocks
3968 .tx_repo
3969 .expect_partial_update()
3970 .withf(|_, update| update.status == Some(TransactionStatus::Failed))
3971 .returning(move |_, update| {
3972 let mut updated_tx = tx_clone.clone();
3973 updated_tx.status = update.status.unwrap();
3974 Ok(updated_tx)
3975 });
3976
3977 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3978
3979 let ctx = StatusCheckContext::new(100, 200, 250, 15, 45, NetworkType::Evm);
3980
3981 let result = evm_transaction
3982 .handle_circuit_breaker_safely(tx, &ctx)
3983 .await;
3984 assert!(result.is_ok());
3985 assert_eq!(result.unwrap().status, TransactionStatus::Failed);
3986 }
3987 }
3988}