feat(freshness): enhance diagnostics and update snapshot structure

- Introduced a new Diagnostics struct to capture transaction visibility state and activity state.
- Updated BuildSnapshot function to return diagnostics alongside snapshot, completeness, and sampling.
- Enhanced test cases to validate the new diagnostics data.
- Updated frontend components to utilize the new diagnostics information for improved user feedback on freshness context.

This change improves the observability of transaction activity and enhances the user experience by providing clearer insights into the freshness of data.

backend/api/freshness/freshness.go

@@ -72,6 +72,22 @@ type Snapshot struct {
 	LatestNonEmptyBlock      Reference `json:"latest_non_empty_block"`
 }
 
+type Diagnostics struct {
+	TxVisibilityState           string       `json:"tx_visibility_state"`
+	ActivityState               string       `json:"activity_state"`
+	Explanation                 string       `json:"explanation,omitempty"`
+	TxLagBlocks                 *int64       `json:"tx_lag_blocks,omitempty"`
+	TxLagSeconds                *int64       `json:"tx_lag_seconds,omitempty"`
+	RecentBlockSampleSize       *int64       `json:"recent_block_sample_size,omitempty"`
+	RecentNonEmptyBlocks        *int64       `json:"recent_non_empty_blocks,omitempty"`
+	RecentTransactions          *int64       `json:"recent_transactions,omitempty"`
+	LatestNonEmptyFromBlockFeed Reference    `json:"latest_non_empty_block_from_block_feed"`
+	Source                      Source       `json:"source"`
+	Confidence                  Confidence   `json:"confidence"`
+	Provenance                  Provenance   `json:"provenance"`
+	Completeness                Completeness `json:"completeness"`
+}
+
 type SummaryCompleteness struct {
 	TransactionsFeed  Completeness `json:"transactions_feed"`
 	BlocksFeed        Completeness `json:"blocks_feed"`
@@ -163,6 +179,49 @@ func classifyMetricPresence[T comparable](value *T) Completeness {
 	return CompletenessComplete
 }
 
+func classifyTxVisibilityState(age *int64) string {
+	if age == nil {
+		return "unavailable"
+	}
+	switch {
+	case *age <= 15*60:
+		return "current"
+	case *age <= 3*60*60:
+		return "lagging"
+	default:
+		return "stale"
+	}
+}
+
+func classifyActivityState(txVisibility string, txLagBlocks, recentTransactions, recentNonEmptyBlocks *int64) (string, string, Completeness) {
+	if txVisibility == "unavailable" {
+		if recentTransactions != nil && *recentTransactions > 0 {
+			return "limited_observability", "Recent blocks show on-chain transaction activity, but indexed transaction freshness is unavailable.", CompletenessPartial
+		}
+		return "limited_observability", "Transaction freshness is unavailable, and recent block activity is limited.", CompletenessUnavailable
+	}
+
+	if recentTransactions != nil && *recentTransactions > 0 {
+		if txLagBlocks != nil && *txLagBlocks > 32 {
+			return "fresh_head_stale_transaction_visibility", "Recent block activity is present closer to the head than the visible indexed transaction feed.", CompletenessPartial
+		}
+		if *recentTransactions <= 3 {
+			return "sparse_activity", "Recent blocks contain only a small amount of transaction activity.", CompletenessComplete
+		}
+		return "active", "Recent blocks contain visible transaction activity close to the head.", CompletenessComplete
+	}
+
+	if recentNonEmptyBlocks != nil && *recentNonEmptyBlocks == 0 {
+		return "quiet_chain", "Recent sampled head blocks are empty, which indicates a quiet chain rather than a broken explorer.", CompletenessComplete
+	}
+
+	if txLagBlocks != nil && *txLagBlocks > 32 {
+		return "fresh_head_stale_transaction_visibility", "The chain head is current, but the indexed transaction feed trails the current tip.", CompletenessPartial
+	}
+
+	return "sparse_activity", "Recent visible transaction activity is limited.", CompletenessComplete
+}
+
 func BuildSnapshot(
 	ctx context.Context,
 	chainID int,
@@ -171,13 +230,22 @@ func BuildSnapshot(
 	now time.Time,
 	averageGasPrice *float64,
 	utilization *float64,
-) (Snapshot, SummaryCompleteness, Sampling, error) {
+) (Snapshot, SummaryCompleteness, Sampling, Diagnostics, error) {
 	snapshot := Snapshot{
 		ChainHead:                unknownReference(ProvenanceRPC),
 		LatestIndexedBlock:       unknownReference(ProvenanceExplorerIndex),
 		LatestIndexedTransaction: unknownReference(ProvenanceTxIndex),
 		LatestNonEmptyBlock:      unknownReference(ProvenanceTxIndex),
 	}
+	diagnostics := Diagnostics{
+		TxVisibilityState:           "unavailable",
+		ActivityState:               "limited_observability",
+		LatestNonEmptyFromBlockFeed: unknownReference(ProvenanceExplorerIndex),
+		Source:                      SourceReported,
+		Confidence:                  ConfidenceMedium,
+		Provenance:                  ProvenanceComposite,
+		Completeness:                CompletenessUnavailable,
+	}
 	issues := map[string]string{}
 
 	if probeHead != nil {
@@ -270,6 +338,84 @@ func BuildSnapshot(
 		issues["latest_non_empty_block"] = err.Error()
 	}
 
+	var latestBlockFeedNonEmptyNumber int64
+	var latestBlockFeedNonEmptyTime time.Time
+	if err := queryRow(ctx,
+		`SELECT b.number, b.timestamp
+		 FROM blocks b
+		 JOIN (
+		   SELECT DISTINCT block_number
+		   FROM transactions
+		   WHERE block_number IS NOT NULL
+		 ) tx_blocks
+		   ON tx_blocks.block_number = b.number
+		 ORDER BY b.number DESC
+		 LIMIT 1`,
+	).Scan(&latestBlockFeedNonEmptyNumber, &latestBlockFeedNonEmptyTime); err == nil {
+		timestamp := timePointer(latestBlockFeedNonEmptyTime)
+		ref := Reference{
+			BlockNumber:  ptrInt64(latestBlockFeedNonEmptyNumber),
+			Timestamp:    timestamp,
+			AgeSeconds:   computeAge(timestamp, now),
+			Source:       SourceDerived,
+			Confidence:   ConfidenceMedium,
+			Provenance:   ProvenanceComposite,
+			Completeness: snapshot.LatestIndexedTransaction.Completeness,
+		}
+		if snapshot.ChainHead.BlockNumber != nil {
+			distance := *snapshot.ChainHead.BlockNumber - latestBlockFeedNonEmptyNumber
+			if distance < 0 {
+				distance = 0
+			}
+			ref.DistanceFromHead = ptrInt64(distance)
+		}
+		diagnostics.LatestNonEmptyFromBlockFeed = ref
+	} else {
+		issues["latest_non_empty_block_from_block_feed"] = err.Error()
+	}
+
+	var recentBlockSampleSize, recentNonEmptyBlocks, recentTransactions int64
+	if err := queryRow(ctx,
+		`SELECT COUNT(*)::bigint,
+		        COUNT(*) FILTER (WHERE COALESCE(tx_counts.tx_count, 0) > 0)::bigint,
+		        COALESCE(SUM(COALESCE(tx_counts.tx_count, 0)), 0)::bigint
+		 FROM (
+		   SELECT number
+		   FROM blocks
+		   ORDER BY number DESC
+		   LIMIT 128
+		 ) recent_blocks
+		 LEFT JOIN (
+		   SELECT block_number, COUNT(*)::bigint AS tx_count
+		   FROM transactions
+		   WHERE block_number IS NOT NULL
+		   GROUP BY block_number
+		 ) tx_counts
+		   ON tx_counts.block_number = recent_blocks.number`,
+	).Scan(&recentBlockSampleSize, &recentNonEmptyBlocks, &recentTransactions); err == nil {
+		diagnostics.RecentBlockSampleSize = ptrInt64(recentBlockSampleSize)
+		diagnostics.RecentNonEmptyBlocks = ptrInt64(recentNonEmptyBlocks)
+		diagnostics.RecentTransactions = ptrInt64(recentTransactions)
+	} else {
+		issues["recent_block_activity"] = err.Error()
+	}
+
+	if snapshot.ChainHead.BlockNumber != nil && snapshot.LatestIndexedTransaction.BlockNumber != nil {
+		lag := *snapshot.ChainHead.BlockNumber - *snapshot.LatestIndexedTransaction.BlockNumber
+		if lag < 0 {
+			lag = 0
+		}
+		diagnostics.TxLagBlocks = ptrInt64(lag)
+	}
+	diagnostics.TxLagSeconds = snapshot.LatestIndexedTransaction.AgeSeconds
+	diagnostics.TxVisibilityState = classifyTxVisibilityState(snapshot.LatestIndexedTransaction.AgeSeconds)
+	diagnostics.ActivityState, diagnostics.Explanation, diagnostics.Completeness = classifyActivityState(
+		diagnostics.TxVisibilityState,
+		diagnostics.TxLagBlocks,
+		diagnostics.RecentTransactions,
+		diagnostics.RecentNonEmptyBlocks,
+	)
+
 	statsGeneratedAt := now.UTC().Format(time.RFC3339)
 	sampling := Sampling{
 		StatsGeneratedAt: ptrString(statsGeneratedAt),
@@ -289,7 +435,7 @@ func BuildSnapshot(
 		UtilizationMetric: classifyMetricPresence(utilization),
 	}
 
-	return snapshot, completeness, sampling, nil
+	return snapshot, completeness, sampling, diagnostics, nil
 }
 
 func ProbeChainHead(ctx context.Context, rpcURL string) (*Reference, error) {

backend/api/freshness/freshness_test.go

@@ -42,6 +42,19 @@ func TestBuildSnapshotHealthyState(t *testing.T) {
 				*dest[1].(*time.Time) = now.Add(-5 * time.Second)
 				return nil
 			}}
+		case 4:
+			return fakeRow{scan: func(dest ...any) error {
+				*dest[0].(*int64) = 198
+				*dest[1].(*time.Time) = now.Add(-5 * time.Second)
+				return nil
+			}}
+		case 5:
+			return fakeRow{scan: func(dest ...any) error {
+				*dest[0].(*int64) = 128
+				*dest[1].(*int64) = 12
+				*dest[2].(*int64) = 34
+				return nil
+			}}
 		default:
 			t.Fatalf("unexpected call %d", call)
 			return nil
@@ -63,13 +76,14 @@ func TestBuildSnapshotHealthyState(t *testing.T) {
 		}, nil
 	}
 
-	snapshot, completeness, sampling, err := BuildSnapshot(context.Background(), 138, queryRow, probe, now, nil, nil)
+	snapshot, completeness, sampling, diagnostics, err := BuildSnapshot(context.Background(), 138, queryRow, probe, now, nil, nil)
 	require.NoError(t, err)
 	require.Equal(t, int64(200), *snapshot.ChainHead.BlockNumber)
 	require.Equal(t, int64(198), *snapshot.LatestIndexedTransaction.BlockNumber)
 	require.Equal(t, int64(2), *snapshot.LatestNonEmptyBlock.DistanceFromHead)
 	require.Equal(t, CompletenessComplete, completeness.TransactionsFeed)
 	require.NotNil(t, sampling.StatsGeneratedAt)
+	require.Equal(t, "active", diagnostics.ActivityState)
 }
 
 func TestBuildSnapshotFreshHeadStaleTransactionVisibility(t *testing.T) {
@@ -97,6 +111,19 @@ func TestBuildSnapshotFreshHeadStaleTransactionVisibility(t *testing.T) {
 				*dest[1].(*time.Time) = now.Add(-(9*time.Hour + 8*time.Minute))
 				return nil
 			}}
+		case 4:
+			return fakeRow{scan: func(dest ...any) error {
+				*dest[0].(*int64) = 3875998
+				*dest[1].(*time.Time) = now.Add(-4 * time.Second)
+				return nil
+			}}
+		case 5:
+			return fakeRow{scan: func(dest ...any) error {
+				*dest[0].(*int64) = 128
+				*dest[1].(*int64) = 3
+				*dest[2].(*int64) = 9
+				return nil
+			}}
 		default:
 			t.Fatalf("unexpected call %d", call)
 			return nil
@@ -118,11 +145,12 @@ func TestBuildSnapshotFreshHeadStaleTransactionVisibility(t *testing.T) {
 		}, nil
 	}
 
-	snapshot, completeness, _, err := BuildSnapshot(context.Background(), 138, queryRow, probe, now, nil, nil)
+	snapshot, completeness, _, diagnostics, err := BuildSnapshot(context.Background(), 138, queryRow, probe, now, nil, nil)
 	require.NoError(t, err)
 	require.Equal(t, int64(15340), *snapshot.LatestNonEmptyBlock.DistanceFromHead)
 	require.Equal(t, CompletenessStale, completeness.TransactionsFeed)
 	require.Equal(t, CompletenessComplete, completeness.BlocksFeed)
+	require.Equal(t, "fresh_head_stale_transaction_visibility", diagnostics.ActivityState)
 }
 
 func TestBuildSnapshotQuietChainButCurrent(t *testing.T) {
@@ -150,6 +178,19 @@ func TestBuildSnapshotQuietChainButCurrent(t *testing.T) {
 				*dest[1].(*time.Time) = now.Add(-512 * time.Second)
 				return nil
 			}}
+		case 4:
+			return fakeRow{scan: func(dest ...any) error {
+				*dest[0].(*int64) = 3874902
+				*dest[1].(*time.Time) = now.Add(-512 * time.Second)
+				return nil
+			}}
+		case 5:
+			return fakeRow{scan: func(dest ...any) error {
+				*dest[0].(*int64) = 128
+				*dest[1].(*int64) = 0
+				*dest[2].(*int64) = 0
+				return nil
+			}}
 		default:
 			t.Fatalf("unexpected call %d", call)
 			return nil
@@ -171,10 +212,11 @@ func TestBuildSnapshotQuietChainButCurrent(t *testing.T) {
 		}, nil
 	}
 
-	snapshot, completeness, _, err := BuildSnapshot(context.Background(), 138, queryRow, probe, now, nil, nil)
+	snapshot, completeness, _, diagnostics, err := BuildSnapshot(context.Background(), 138, queryRow, probe, now, nil, nil)
 	require.NoError(t, err)
 	require.Equal(t, int64(98), *snapshot.LatestNonEmptyBlock.DistanceFromHead)
 	require.Equal(t, CompletenessComplete, completeness.TransactionsFeed)
+	require.Equal(t, "quiet_chain", diagnostics.ActivityState)
 }
 
 func TestBuildSnapshotUnknownFieldsRemainNullSafe(t *testing.T) {
@@ -184,9 +226,10 @@ func TestBuildSnapshotUnknownFieldsRemainNullSafe(t *testing.T) {
 		}}
 	}
 
-	snapshot, completeness, sampling, err := BuildSnapshot(context.Background(), 138, queryRow, nil, time.Now().UTC(), nil, nil)
+	snapshot, completeness, sampling, diagnostics, err := BuildSnapshot(context.Background(), 138, queryRow, nil, time.Now().UTC(), nil, nil)
 	require.NoError(t, err)
 	require.Nil(t, snapshot.ChainHead.BlockNumber)
 	require.Equal(t, CompletenessUnavailable, completeness.TransactionsFeed)
 	require.NotNil(t, sampling.StatsGeneratedAt)
+	require.Equal(t, "limited_observability", diagnostics.ActivityState)
 }

backend/api/rest/stats.go

@@ -25,6 +25,7 @@ type explorerStats struct {
 	Freshness                    freshness.Snapshot            `json:"freshness"`
 	Completeness                 freshness.SummaryCompleteness `json:"completeness"`
 	Sampling                     freshness.Sampling            `json:"sampling"`
+	Diagnostics                  freshness.Diagnostics         `json:"diagnostics"`
 }
 
 type explorerGasPrices struct {
@@ -160,7 +161,7 @@ func loadExplorerStats(ctx context.Context, chainID int, queryRow statsQueryFunc
 	}
 
 	rpcURL := strings.TrimSpace(os.Getenv("RPC_URL"))
-	snapshot, completeness, sampling, err := freshness.BuildSnapshot(
+	snapshot, completeness, sampling, diagnostics, err := freshness.BuildSnapshot(
 		ctx,
 		chainID,
 		queryRow,
@@ -185,6 +186,7 @@ func loadExplorerStats(ctx context.Context, chainID int, queryRow statsQueryFunc
 	stats.Freshness = snapshot
 	stats.Completeness = completeness
 	stats.Sampling = sampling
+	stats.Diagnostics = diagnostics
 
 	return stats, nil
 }

backend/api/rest/stats_internal_test.go

@@ -76,6 +76,13 @@ func TestLoadExplorerStatsReturnsValues(t *testing.T) {
 				case 11:
 					*dest[0].(*int64) = 40
 					*dest[1].(*time.Time) = time.Now().Add(-5 * time.Second)
+				case 12:
+					*dest[0].(*int64) = 42
+					*dest[1].(*time.Time) = time.Now().Add(-3 * time.Second)
+				case 13:
+					*dest[0].(*int64) = 128
+					*dest[1].(*int64) = 10
+					*dest[2].(*int64) = 22
 				default:
 					t.Fatalf("unexpected query call %d", call)
 				}
@@ -102,6 +109,8 @@ func TestLoadExplorerStatsReturnsValues(t *testing.T) {
 	require.NotNil(t, stats.Freshness.ChainHead.BlockNumber)
 	require.Equal(t, int64(40), *stats.Freshness.LatestIndexedTransaction.BlockNumber)
 	require.Equal(t, int64(4), *stats.Freshness.LatestNonEmptyBlock.DistanceFromHead)
+	require.Equal(t, "active", stats.Diagnostics.ActivityState)
+	require.Equal(t, int64(4), *stats.Diagnostics.TxLagBlocks)
 	require.Equal(t, "reported", string(stats.Freshness.ChainHead.Source))
 	require.Equal(t, freshness.CompletenessComplete, stats.Completeness.GasMetrics)
 	require.Equal(t, freshness.CompletenessComplete, stats.Completeness.UtilizationMetric)

backend/api/rest/track_routes.go

@@ -50,12 +50,12 @@ func (s *Server) SetupTrackRoutes(mux *http.ServeMux, authMiddleware *middleware
 	}
 
 	rpcGateway := gateway.NewRPCGateway(rpcURL, cache, rateLimiter)
-	track1Server := track1.NewServer(rpcGateway, func(ctx context.Context) (*freshness.Snapshot, *freshness.SummaryCompleteness, *freshness.Sampling, error) {
+	track1Server := track1.NewServer(rpcGateway, func(ctx context.Context) (*freshness.Snapshot, *freshness.SummaryCompleteness, *freshness.Sampling, *freshness.Diagnostics, error) {
 		if s.db == nil {
-			return nil, nil, nil, nil
+			return nil, nil, nil, nil, nil
 		}
 		now := time.Now().UTC()
-		snapshot, completeness, sampling, err := freshness.BuildSnapshot(
+		snapshot, completeness, sampling, diagnostics, err := freshness.BuildSnapshot(
 			ctx,
 			s.chainID,
 			s.db.QueryRow,
@@ -67,9 +67,9 @@ func (s *Server) SetupTrackRoutes(mux *http.ServeMux, authMiddleware *middleware
 			nil,
 		)
 		if err != nil {
-			return nil, nil, nil, err
+			return nil, nil, nil, nil, err
 		}
-		return &snapshot, &completeness, &sampling, nil
+		return &snapshot, &completeness, &sampling, &diagnostics, nil
 	})
 
 	// Track 1 routes (public, optional auth)

backend/api/track1/bridge_status_data.go

@@ -132,7 +132,7 @@ func (s *Server) BuildBridgeStatusData(ctx context.Context) map[string]interface
 		data["operator_verify"] = ov
 	}
 	if s.freshnessLoader != nil {
-		if snapshot, completeness, sampling, err := s.freshnessLoader(ctx); err == nil && snapshot != nil {
+		if snapshot, completeness, sampling, diagnostics, err := s.freshnessLoader(ctx); err == nil && snapshot != nil {
 			subsystems := map[string]interface{}{
 				"rpc_head": map[string]interface{}{
 					"status":       chainStatusFromProbe(p138),
@@ -194,6 +194,9 @@ func (s *Server) BuildBridgeStatusData(ctx context.Context) map[string]interface
 			data["freshness"] = snapshot
 			data["subsystems"] = subsystems
 			data["sampling"] = sampling
+			if diagnostics != nil {
+				data["diagnostics"] = diagnostics
+			}
 			data["mode"] = map[string]interface{}{
 				"kind":        modeKind,
 				"updated_at":  now,

backend/api/track1/ccip_health_test.go

@@ -147,7 +147,7 @@ func TestBuildBridgeStatusDataIncludesCCIPRelay(t *testing.T) {
 	t.Setenv("MISSION_CONTROL_CCIP_JSON", "")
 
 	s := &Server{
-		freshnessLoader: func(context.Context) (*freshness.Snapshot, *freshness.SummaryCompleteness, *freshness.Sampling, error) {
+		freshnessLoader: func(context.Context) (*freshness.Snapshot, *freshness.SummaryCompleteness, *freshness.Sampling, *freshness.Diagnostics, error) {
 			now := time.Now().UTC().Format(time.RFC3339)
 			head := int64(16)
 			txBlock := int64(12)
@@ -187,6 +187,14 @@ func TestBuildBridgeStatusDataIncludesCCIPRelay(t *testing.T) {
 					BlocksFeed:       freshness.CompletenessComplete,
 				},
 				&freshness.Sampling{StatsGeneratedAt: &now},
+				&freshness.Diagnostics{
+					TxVisibilityState: "lagging",
+					ActivityState:     "fresh_head_stale_transaction_visibility",
+					Source:            freshness.SourceReported,
+					Confidence:        freshness.ConfidenceMedium,
+					Provenance:        freshness.ProvenanceComposite,
+					Completeness:      freshness.CompletenessPartial,
+				},
 				nil
 		},
 	}
@@ -201,6 +209,7 @@ func TestBuildBridgeStatusDataIncludesCCIPRelay(t *testing.T) {
 	require.True(t, ok)
 	require.Equal(t, true, probe["ok"])
 	require.Contains(t, got, "freshness")
+	require.Contains(t, got, "diagnostics")
 	require.Contains(t, got, "subsystems")
 	require.Contains(t, got, "mode")
 }
@@ -245,8 +254,8 @@ func TestBuildBridgeStatusDataDegradesWhenNamedRelayFails(t *testing.T) {
 	t.Setenv("CCIP_RELAY_HEALTH_URLS", "mainnet="+mainnet.URL+"/healthz,bsc="+bad.URL+"/healthz")
 
 	s := &Server{
-		freshnessLoader: func(context.Context) (*freshness.Snapshot, *freshness.SummaryCompleteness, *freshness.Sampling, error) {
-			return nil, nil, nil, nil
+		freshnessLoader: func(context.Context) (*freshness.Snapshot, *freshness.SummaryCompleteness, *freshness.Sampling, *freshness.Diagnostics, error) {
+			return nil, nil, nil, nil, nil
 		},
 	}
 	got := s.BuildBridgeStatusData(context.Background())

backend/api/track1/endpoints.go

@@ -21,13 +21,13 @@ var track1HashPattern = regexp.MustCompile(`^0x[a-fA-F0-9]{64}$`)
 // Server handles Track 1 endpoints (uses RPC gateway from lib)
 type Server struct {
 	rpcGateway      *gateway.RPCGateway
-	freshnessLoader func(ctx context.Context) (*freshness.Snapshot, *freshness.SummaryCompleteness, *freshness.Sampling, error)
+	freshnessLoader func(ctx context.Context) (*freshness.Snapshot, *freshness.SummaryCompleteness, *freshness.Sampling, *freshness.Diagnostics, error)
 }
 
 // NewServer creates a new Track 1 server
 func NewServer(
 	rpcGateway *gateway.RPCGateway,
-	freshnessLoader func(ctx context.Context) (*freshness.Snapshot, *freshness.SummaryCompleteness, *freshness.Sampling, error),
+	freshnessLoader func(ctx context.Context) (*freshness.Snapshot, *freshness.SummaryCompleteness, *freshness.Sampling, *freshness.Diagnostics, error),
 ) *Server {
 	return &Server{
 		rpcGateway:      rpcGateway,