Event-based Metrics (#147)

This PR adds a number of metrics for errors in
parsing/tx handling -- especially for our squads monitoring. It also
removes the old once-per-epoch flush in favor of real time events sent
to influxdb. NOTE: This means that any dashboard currently reliant on
the per-epoch data will break. Though I assume this is fine, as we
weren't measuring much to begin with.

Author: hash-envy

.env.example

@@ -1,5 +1,6 @@
 RUST_LOG=info
-ENDPOINT=
+YELLOWSTONE_URL=
+RPC_URL=
 X_TOKEN=
 FAILED=true
 ACCOUNT_INCLUDE=SPoo1Ku8WFXoNDMHPsrGSTSG1Y47rzgn41SLUNakuHy,Vau1t6sLNxnzB7ZDsef8TLbPLfyZMYXH8WTNqUdm9g8

docker-compose.yaml

@@ -8,12 +8,14 @@ services:
     environment:
       - RUST_LOG=${RUST_LOG:-debug}
       - YELLOWSTONE_URL=${YELLOWSTONE_URL}
+      - RPC_URL=${RPC_URL}
       - X_TOKEN=${X_TOKEN}
       - ACCOUNT_INCLUDE=${ACCOUNT_INCLUDE}
       - CONFIG_FILE=${CONFIG_FILE}
       - SOLANA_METRICS_CONFIG=${SOLANA_METRICS_CONFIG}
       - SLACK_WEBHOOK_URL={SLACK_WEBHOOK_URL}
       - STAKE_POOL_ALERTS_SLACK_WEBHOOK_URL={STAKE_POOL_ALERTS_SLACK_WEBHOOK_URL}
+      - SQUADS_ALERTS_SLACK_WEBHOOK_URL={SQUADS_ALERTS_SLACK_WEBHOOK_URL}
       - DISCORD_WEBHOOK_URL={DISCORD_WEBHOOK_URL}
       - TELEGRAM_BOT_TOKEN={TELEGRAM_BOT_TOKEN}
       - TELEGRAM_CHAT_ID={TELEGRAM_CHAT_ID}

jito-bell/src/bin/main.rs

@@ -71,7 +71,7 @@ async fn main() -> anyhow::Result<()> {
 
     let commitment = CommitmentConfig::confirmed();
     let mut handler = JitoBellHandler::new(
-        args.yellowstone_url.clone(),
+        args.rpc_url.clone(),
         commitment,
         args.config_file,
         subscribe_option,

jito-bell/src/cli_args.rs

@@ -10,6 +10,10 @@ pub struct Args {
     /// Yellowstone gRPC endpoint URL
     pub yellowstone_url: String,
 
+    #[clap(long, env = "RPC_URL")]
+    /// Solana JSON-RPC endpoint URL
+    pub rpc_url: String,
+
     #[clap(long, env = "X_TOKEN")]
     pub x_token: Option<String>,
 
@@ -53,6 +57,10 @@ pub struct Args {
     #[clap(long, env)]
     pub stakenet_event_alerts_slack_webhook_url: Option<String>,
 
+    /// Slack webhook URL for Squads Alerts
+    #[clap(long, env)]
+    pub squads_alerts_slack_webhook_url: Option<String>,
+
     /// Discord webhook URL
     #[clap(long, env)]
     pub discord_webhook_url: Option<String>,

jito-bell/src/config.rs

@@ -12,18 +12,10 @@ pub struct JitoBellConfig {
     /// Block explorer url
     pub explorer_url: String,
 
-    /// Squads app URL template
-    #[serde(default = "default_squads_app_url_template")]
-    pub squads_app_url_template: String,
-
     /// Message Templates
     pub message_templates: HashMap<String, String>,
 }
 
-fn default_squads_app_url_template() -> String {
-    "https://app.squads.so/squads/{{multisig}}/transactions/{{transaction}}".to_string()
-}
-
 impl JitoBellConfig {
     /// Get a message template by name, falling back to default
     pub fn get_template(&self, name: &str) -> Option<&String> {
@@ -52,11 +44,6 @@ mod tests {
 impl std::fmt::Display for JitoBellConfig {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         writeln!(f, "Explorer URL: {}", self.explorer_url)?;
-        writeln!(
-            f,
-            "Squads App URL Template: {}",
-            self.squads_app_url_template
-        )?;
 
         writeln!(f, "Message Templates:")?;
         for (name, template) in &self.message_templates {

jito-bell/src/handlers/mod.rs

@@ -13,7 +13,7 @@ mod stake_pool;
 mod vault;
 
 use borsh::BorshDeserialize;
-use log::debug;
+use log::{debug, warn};
 use solana_rpc_client::nonblocking::rpc_client::RpcClient;
 use solana_sdk::{program_pack::Pack, pubkey::Pubkey};
 use spl_token::state::Mint;
@@ -123,23 +123,37 @@ pub(crate) async fn send_notification(
                 }
             }
             InstructionParser::SquadsV3(ix) => match ix {
-                SquadsV3Program::CreateTransaction { .. } => {
+                SquadsV3Program::CreateTransaction { multisig, .. } => {
+                    handler.increment_squads_parsed();
                     if let Some(instruction) =
                         handler.get_instruction_config(ProgramName::SquadsV3, ix)
                     {
                         squads_v3::handle_squads_v3_program(handler, parser, ix, &instruction)
                             .await?;
+                    } else {
+                        handler.increment_squads_no_config();
+                        warn!(
+                            "SquadsV3: CreateTransaction from multisig {} has no handler config",
+                            multisig
+                        );
                     }
                 }
                 SquadsV3Program::ActivateTransaction { ix: _ } => {}
             },
             InstructionParser::SquadsV4(ix) => match ix {
-                SquadsV4Program::ProposalCreate { .. } => {
+                SquadsV4Program::ProposalCreate { multisig, .. } => {
+                    handler.increment_squads_parsed();
                     if let Some(instruction) =
                         handler.get_instruction_config(ProgramName::SquadsV4, ix)
                     {
                         squads_v4::handle_squads_v4_program(handler, parser, ix, &instruction)
                             .await?;
+                    } else {
+                        handler.increment_squads_no_config();
+                        warn!(
+                            "SquadsV4: ProposalCreate from multisig {} has no handler config",
+                            multisig
+                        );
                     }
                 }
                 SquadsV4Program::ProposalActivate { ix: _ } => {}

jito-bell/src/handlers/squads_common.rs

@@ -1,9 +1,17 @@
 //! Shared types for Squads v3/v4 notification handling.
 
-use solana_sdk::pubkey::Pubkey;
+use base64::{engine::general_purpose::STANDARD, Engine as _};
+use solana_sdk::{pubkey, pubkey::Pubkey};
+
+const SQUADS_V4_URL: &str =
+    "https://app.squads.so/squads/{{multisig}}/transactions/{{transaction}}";
+const SQUADS_V4_PROGRAM_ID: Pubkey = pubkey!("SQDS4ep65T869zMMBKyuUq6aD6EgTu8psMjkvj52pCf");
+const SQUADS_V4_SEED_PREFIX: &[u8] = b"multisig";
+const SQUADS_V4_SEED_VAULT: &[u8] = b"vault";
+const SQUADS_V4_SEED_TRANSACTION: &[u8] = b"transaction";
 
 #[derive(Clone, Copy, Debug)]
-pub(crate) enum SquadsContext {
+pub enum SquadsContext {
     V3Transaction {
         multisig: Pubkey,
         transaction: Pubkey,
@@ -52,10 +60,22 @@ impl SquadsContext {
 
     fn transaction_template_value(self) -> String {
         match self {
-            Self::V3Transaction { transaction, .. } => transaction.to_string(),
+            Self::V3Transaction { transaction, .. } => STANDARD.encode(transaction.to_string()),
             Self::V4Proposal {
-                transaction_index, ..
-            } => transaction_index.to_string(),
+                multisig,
+                transaction_index,
+                ..
+            } => Pubkey::find_program_address(
+                &[
+                    SQUADS_V4_SEED_PREFIX,
+                    multisig.as_ref(),
+                    SQUADS_V4_SEED_TRANSACTION,
+                    &transaction_index.to_le_bytes(),
+                ],
+                &SQUADS_V4_PROGRAM_ID,
+            )
+            .0
+            .to_string(),
         }
     }
 
@@ -75,10 +95,99 @@ impl SquadsContext {
         }
     }
 
-    pub(crate) fn squads_url(self, template: &str) -> String {
-        template
-            .replace("{{multisig}}", &self.multisig().to_string())
-            .replace("{{transaction}}", &self.transaction_template_value())
-            .replace("{{proposal}}", &self.proposal_template_value())
+    fn multisig_template_value(self) -> String {
+        match self {
+            // V3 URL has no {{multisig}} placeholder; value unused but kept consistent.
+            Self::V3Transaction { .. } => self.multisig().to_string(),
+            // The v4 app route uses the squad's vault PDA, not the multisig account.
+            Self::V4Proposal { multisig, .. } => Pubkey::find_program_address(
+                &[
+                    SQUADS_V4_SEED_PREFIX,
+                    multisig.as_ref(),
+                    SQUADS_V4_SEED_VAULT,
+                    &[0],
+                ],
+                &SQUADS_V4_PROGRAM_ID,
+            )
+            .0
+            .to_string(),
+        }
+    }
+
+    pub(crate) fn squads_url(self, _explorer_url: &str, _transaction_signature: &str) -> String {
+        match self {
+            Self::V3Transaction {
+                multisig,
+                transaction,
+            } => format!(
+                "https://v3.squads.so/transactions/{}/tx/{}",
+                STANDARD.encode(multisig.to_string()),
+                transaction,
+            ),
+            Self::V4Proposal { .. } => SQUADS_V4_URL
+                .replace("{{multisig}}", &self.multisig_template_value())
+                .replace("{{transaction}}", &self.transaction_template_value())
+                .replace("{{proposal}}", &self.proposal_template_value()),
+        }
+    }
+
+    pub fn build_slack_payload(
+        self,
+        description: &str,
+        transaction_signature: &str,
+        explorer_url: &str,
+    ) -> serde_json::Value {
+        let squads_url = self.squads_url(explorer_url, transaction_signature);
+        let mut fields = vec![
+            serde_json::json!({
+                "type": "mrkdwn",
+                "text": format!("*Squads:* <{}|{}>", squads_url, self.link_label())
+            }),
+            serde_json::json!({
+                "type": "mrkdwn",
+                "text": format!(
+                    "*Transaction:* <{}/tx/{}|View on Explorer>",
+                    explorer_url, transaction_signature
+                )
+            }),
+            serde_json::json!({
+                "type": "mrkdwn",
+                "text": format!("*Multisig:* `{}`", self.multisig())
+            }),
+            serde_json::json!({
+                "type": "mrkdwn",
+                "text": format!("*{}:* `{}`", self.account_label(), self.account())
+            }),
+        ];
+
+        if let Some(transaction_index) = self.transaction_index_field() {
+            fields.push(serde_json::json!({
+                "type": "mrkdwn",
+                "text": format!("*Transaction Index:* `{}`", transaction_index)
+            }));
+        }
+
+        serde_json::json!({
+            "blocks": [
+                {
+                    "type": "header",
+                    "text": {
+                        "type": "plain_text",
+                        "text": self.header()
+                    }
+                },
+                {
+                    "type": "section",
+                    "text": {
+                        "type": "mrkdwn",
+                        "text": format!("*Description:* {}", description)
+                    }
+                },
+                {
+                    "type": "section",
+                    "fields": fields
+                }
+            ]
+        })
     }
 }

jito-bell/src/ix_parser/squads_v3.rs

@@ -1,3 +1,4 @@
+use log::debug;
 use solana_sdk::{
     hash::hash,
     instruction::{AccountMeta, Instruction},
@@ -39,15 +40,26 @@ impl SquadsV3Program {
     pub fn parse_squads_v3_program<T: ParsableInstruction>(
         instruction: &T,
         account_keys: &[Pubkey],
+        squads_parse_errors: &mut u64,
     ) -> Option<SquadsV3Program> {
         let discriminator: [u8; 8] = instruction.data().get(..8)?.try_into().ok()?;
 
         if discriminator == anchor_discriminator(Self::CREATE_TRANSACTION) {
-            return Self::parse_create_transaction_ix(instruction, account_keys);
+            let result = Self::parse_create_transaction_ix(instruction, account_keys);
+            if result.is_none() {
+                *squads_parse_errors += 1;
+                debug!("SquadsV3: matched create_transaction discriminator but failed to parse (short data or invalid account index)");
+            }
+            return result;
         }
 
         if discriminator == anchor_discriminator(Self::ACTIVATE_TRANSACTION) {
-            return Self::parse_activate_transaction_ix(instruction, account_keys);
+            let result = Self::parse_activate_transaction_ix(instruction, account_keys);
+            if result.is_none() {
+                *squads_parse_errors += 1;
+                debug!("SquadsV3: matched activate_transaction discriminator but failed to parse (short data or invalid account index)");
+            }
+            return result;
         }
 
         None
@@ -130,7 +142,7 @@ mod tests {
         data.extend_from_slice(&[1, 2, 3, 4]);
         let instruction = create_compiled_instruction(4, vec![0, 1, 2, 3], data.clone());
 
-        let parsed = SquadsV3Program::parse_squads_v3_program(&instruction, &account_keys);
+        let parsed = SquadsV3Program::parse_squads_v3_program(&instruction, &account_keys, &mut 0);
 
         let Some(SquadsV3Program::CreateTransaction {
             multisig,
@@ -150,7 +162,7 @@ mod tests {
         data.extend_from_slice(&[1, 2, 3, 4]);
         let instruction = create_compiled_instruction(3, vec![0, 1, 2], data.clone());
 
-        let parsed = SquadsV3Program::parse_squads_v3_program(&instruction, &account_keys);
+        let parsed = SquadsV3Program::parse_squads_v3_program(&instruction, &account_keys, &mut 0);
 
         let Some(SquadsV3Program::ActivateTransaction { ix }) = parsed else {
             panic!("Expected ActivateTransaction variant");
@@ -167,12 +179,31 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_create_transaction_with_missing_accounts_increments_error_counter() {
+        let account_keys = create_test_pubkeys(2);
+        let mut data = anchor_discriminator("create_transaction").to_vec();
+        data.extend_from_slice(&[1, 2, 3, 4]);
+        // No accounts provided — parse_create_transaction_ix will return None
+        let instruction = create_compiled_instruction(1, vec![], data);
+
+        let mut errors = 0u64;
+        let parsed =
+            SquadsV3Program::parse_squads_v3_program(&instruction, &account_keys, &mut errors);
+
+        assert!(parsed.is_none());
+        assert_eq!(
+            errors, 1,
+            "known discriminator with missing accounts should increment parse error counter"
+        );
+    }
+
     #[test]
     fn test_unknown_data_returns_none() {
         let account_keys = create_test_pubkeys(3);
         let instruction = create_compiled_instruction(0, vec![0, 1, 2], vec![0; 8]);
 
-        let parsed = SquadsV3Program::parse_squads_v3_program(&instruction, &account_keys);
+        let parsed = SquadsV3Program::parse_squads_v3_program(&instruction, &account_keys, &mut 0);
 
         assert!(parsed.is_none());
     }
@@ -182,7 +213,7 @@ mod tests {
         let account_keys = create_test_pubkeys(3);
         let instruction = create_compiled_instruction(0, vec![0, 1, 2], vec![0; 7]);
 
-        let parsed = SquadsV3Program::parse_squads_v3_program(&instruction, &account_keys);
+        let parsed = SquadsV3Program::parse_squads_v3_program(&instruction, &account_keys, &mut 0);
 
         assert!(parsed.is_none());
     }