Freshness diagnostics API, UI trust notes, mission control/stats updates, and deploy scripts.

Made-with: Cursor

backend/api/rest/mission_control.go

@@ -48,6 +48,46 @@ func tokenAggregationBase() string {
 	return ""
 }
 
+func looksLikeGenericUpstreamErrorPayload(body []byte) bool {
+	if len(bytes.TrimSpace(body)) == 0 {
+		return false
+	}
+
+	var payload map[string]any
+	if err := json.Unmarshal(body, &payload); err != nil {
+		return false
+	}
+
+	errValue, ok := payload["error"].(string)
+	if !ok || strings.TrimSpace(errValue) == "" {
+		return false
+	}
+
+	if _, ok := payload["pools"]; ok {
+		return false
+	}
+	if _, ok := payload["tokens"]; ok {
+		return false
+	}
+	if _, ok := payload["data"]; ok {
+		return false
+	}
+	if _, ok := payload["chains"]; ok {
+		return false
+	}
+	if _, ok := payload["tree"]; ok {
+		return false
+	}
+	if _, ok := payload["quote"]; ok {
+		return false
+	}
+	if status, ok := payload["status"].(string); ok && strings.EqualFold(status, "healthy") {
+		return false
+	}
+
+	return true
+}
+
 func blockscoutInternalBase() string {
 	u := strings.TrimSpace(os.Getenv("BLOCKSCOUT_INTERNAL_URL"))
 	if u == "" {
@@ -156,6 +196,15 @@ func (s *Server) handleMissionControlLiquidityTokenPath(w http.ResponseWriter, r
 	if ctype == "" {
 		ctype = "application/json"
 	}
+	isGenericSuccessError := resp.StatusCode >= 200 && resp.StatusCode < 300 && looksLikeGenericUpstreamErrorPayload(body)
+	if isGenericSuccessError {
+		atomic.AddUint64(&missionControlMetrics.liquidityUpstreamFailure, 1)
+		log.Printf("mission_control liquidity_proxy addr=%s chain=%s cache=miss upstream_status=%d generic_error_envelope=true", strings.ToLower(addr), chain, resp.StatusCode)
+		w.Header().Set("Content-Type", ctype)
+		w.WriteHeader(http.StatusBadGateway)
+		_, _ = w.Write(body)
+		return
+	}
 	if resp.StatusCode == http.StatusOK {
 		liquidityPoolsCache.Store(cacheKey, liquidityCacheEntry{
 			body:  body,

backend/api/rest/mission_control_test.go

@@ -98,6 +98,37 @@ func TestHandleMissionControlLiquidityTokenPathBypassesCacheWhenRequested(t *tes
 	require.Equal(t, 2, hitCount, "refresh=1 should force a fresh upstream read")
 }
 
+func TestHandleMissionControlLiquidityTokenPathTreatsGenericSuccessErrorEnvelopeAsBadGateway(t *testing.T) {
+	resetMissionControlTestGlobals()
+
+	var hitCount int
+	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hitCount++
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte(`{"error":"Internal server error"}`))
+	}))
+	defer upstream.Close()
+
+	t.Setenv("TOKEN_AGGREGATION_BASE_URL", upstream.URL)
+	t.Setenv("CHAIN_ID", "138")
+
+	s := NewServer(nil, 138)
+	path := "/api/v1/mission-control/liquidity/token/0x93E66202A11B1772E55407B32B44e5Cd8eda7f22/pools"
+
+	w1 := httptest.NewRecorder()
+	s.handleMissionControlLiquidityTokenPath(w1, httptest.NewRequest(http.MethodGet, path, nil))
+	require.Equal(t, http.StatusBadGateway, w1.Code)
+	require.Equal(t, "miss", w1.Header().Get("X-Mission-Control-Cache"))
+	require.JSONEq(t, `{"error":"Internal server error"}`, w1.Body.String())
+
+	w2 := httptest.NewRecorder()
+	s.handleMissionControlLiquidityTokenPath(w2, httptest.NewRequest(http.MethodGet, path, nil))
+	require.Equal(t, http.StatusBadGateway, w2.Code)
+	require.Equal(t, "miss", w2.Header().Get("X-Mission-Control-Cache"))
+	require.Equal(t, 2, hitCount, "generic error envelopes must not be cached as success")
+}
+
 func TestHandleMissionControlBridgeTraceLabelsFromRegistry(t *testing.T) {
 	resetMissionControlTestGlobals()
 

backend/api/rest/stats.go

@@ -2,62 +2,190 @@ package rest
 
 import (
 	"context"
+	"database/sql"
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"os"
+	"strings"
 	"time"
 
-	"github.com/jackc/pgx/v5"
+	"github.com/explorer/backend/api/freshness"
 )
 
 type explorerStats struct {
-	TotalBlocks       int64 `json:"total_blocks"`
-	TotalTransactions int64 `json:"total_transactions"`
-	TotalAddresses    int64 `json:"total_addresses"`
-	LatestBlock       int64 `json:"latest_block"`
+	TotalBlocks                  int64                         `json:"total_blocks"`
+	TotalTransactions            int64                         `json:"total_transactions"`
+	TotalAddresses               int64                         `json:"total_addresses"`
+	LatestBlock                  int64                         `json:"latest_block"`
+	AverageBlockTime             *float64                      `json:"average_block_time,omitempty"`
+	GasPrices                    *explorerGasPrices            `json:"gas_prices,omitempty"`
+	NetworkUtilizationPercentage *float64                      `json:"network_utilization_percentage,omitempty"`
+	TransactionsToday            *int64                        `json:"transactions_today,omitempty"`
+	Freshness                    freshness.Snapshot            `json:"freshness"`
+	Completeness                 freshness.SummaryCompleteness `json:"completeness"`
+	Sampling                     freshness.Sampling            `json:"sampling"`
 }
 
-type statsQueryFunc func(ctx context.Context, sql string, args ...any) pgx.Row
+type explorerGasPrices struct {
+	Average *float64 `json:"average,omitempty"`
+}
+
+type statsQueryFunc = freshness.QueryRowFunc
+
+func queryNullableFloat64(ctx context.Context, queryRow statsQueryFunc, query string, args ...any) (*float64, error) {
+	var value sql.NullFloat64
+	if err := queryRow(ctx, query, args...).Scan(&value); err != nil {
+		return nil, err
+	}
+	if !value.Valid {
+		return nil, nil
+	}
+	return &value.Float64, nil
+}
+
+func queryNullableInt64(ctx context.Context, queryRow statsQueryFunc, query string, args ...any) (*int64, error) {
+	var value sql.NullInt64
+	if err := queryRow(ctx, query, args...).Scan(&value); err != nil {
+		return nil, err
+	}
+	if !value.Valid {
+		return nil, nil
+	}
+	return &value.Int64, nil
+}
 
 func loadExplorerStats(ctx context.Context, chainID int, queryRow statsQueryFunc) (explorerStats, error) {
 	var stats explorerStats
+	_ = chainID
 
 	if err := queryRow(ctx,
-		`SELECT COUNT(*) FROM blocks WHERE chain_id = $1`,
-		chainID,
+		`SELECT COUNT(*) FROM blocks`,
 	).Scan(&stats.TotalBlocks); err != nil {
 		return explorerStats{}, fmt.Errorf("query total blocks: %w", err)
 	}
 
 	if err := queryRow(ctx,
-		`SELECT COUNT(*) FROM transactions WHERE chain_id = $1`,
-		chainID,
+		`SELECT COUNT(*) FROM transactions WHERE block_hash IS NOT NULL`,
 	).Scan(&stats.TotalTransactions); err != nil {
 		return explorerStats{}, fmt.Errorf("query total transactions: %w", err)
 	}
 
 	if err := queryRow(ctx,
 		`SELECT COUNT(*) FROM (
-			SELECT from_address AS address
+			SELECT from_address_hash AS address
 			FROM transactions
-			WHERE chain_id = $1 AND from_address IS NOT NULL AND from_address <> ''
+			WHERE from_address_hash IS NOT NULL
 			UNION
-			SELECT to_address AS address
+			SELECT to_address_hash AS address
 			FROM transactions
-			WHERE chain_id = $1 AND to_address IS NOT NULL AND to_address <> ''
+			WHERE to_address_hash IS NOT NULL
 		) unique_addresses`,
-		chainID,
 	).Scan(&stats.TotalAddresses); err != nil {
 		return explorerStats{}, fmt.Errorf("query total addresses: %w", err)
 	}
 
 	if err := queryRow(ctx,
-		`SELECT COALESCE(MAX(number), 0) FROM blocks WHERE chain_id = $1`,
-		chainID,
+		`SELECT COALESCE(MAX(number), 0) FROM blocks`,
 	).Scan(&stats.LatestBlock); err != nil {
 		return explorerStats{}, fmt.Errorf("query latest block: %w", err)
 	}
 
+	statsIssues := map[string]string{}
+
+	averageBlockTime, err := queryNullableFloat64(ctx, queryRow,
+		`SELECT CASE
+		     WHEN COUNT(*) >= 2
+		       THEN (EXTRACT(EPOCH FROM (MAX(timestamp) - MIN(timestamp))) * 1000.0) / NULLIF(COUNT(*) - 1, 0)
+		     ELSE NULL
+		   END
+		 FROM (
+		   SELECT timestamp
+		   FROM blocks
+		   ORDER BY number DESC
+		   LIMIT 100
+		 ) recent_blocks`,
+	)
+	if err != nil {
+		statsIssues["average_block_time"] = err.Error()
+	} else {
+		stats.AverageBlockTime = averageBlockTime
+	}
+
+	averageGasPrice, err := queryNullableFloat64(ctx, queryRow,
+		`SELECT AVG(gas_price_wei)::double precision / 1000000000.0
+		 FROM (
+		   SELECT gas_price AS gas_price_wei
+		   FROM transactions
+		   WHERE block_hash IS NOT NULL
+		     AND gas_price IS NOT NULL
+		   ORDER BY block_number DESC, "index" DESC
+		   LIMIT 1000
+		 ) recent_transactions`,
+	)
+	if err != nil {
+		statsIssues["average_gas_price"] = err.Error()
+	} else if averageGasPrice != nil {
+		stats.GasPrices = &explorerGasPrices{Average: averageGasPrice}
+	}
+
+	networkUtilization, err := queryNullableFloat64(ctx, queryRow,
+		`SELECT AVG((gas_used::double precision / NULLIF(gas_limit, 0)) * 100.0)
+		 FROM (
+		   SELECT gas_used, gas_limit
+		   FROM blocks
+		   WHERE gas_limit IS NOT NULL
+		     AND gas_limit > 0
+		   ORDER BY number DESC
+		   LIMIT 100
+		 ) recent_blocks`,
+	)
+	if err != nil {
+		statsIssues["network_utilization_percentage"] = err.Error()
+	} else {
+		stats.NetworkUtilizationPercentage = networkUtilization
+	}
+
+	transactionsToday, err := queryNullableInt64(ctx, queryRow,
+		`SELECT COUNT(*)::bigint
+		 FROM transactions t
+		 JOIN blocks b
+		   ON b.number = t.block_number
+		 WHERE b.timestamp >= NOW() - INTERVAL '24 hours'`,
+	)
+	if err != nil {
+		statsIssues["transactions_today"] = err.Error()
+	} else {
+		stats.TransactionsToday = transactionsToday
+	}
+
+	rpcURL := strings.TrimSpace(os.Getenv("RPC_URL"))
+	snapshot, completeness, sampling, err := freshness.BuildSnapshot(
+		ctx,
+		chainID,
+		queryRow,
+		func(ctx context.Context) (*freshness.Reference, error) {
+			return freshness.ProbeChainHead(ctx, rpcURL)
+		},
+		time.Now().UTC(),
+		averageGasPrice,
+		networkUtilization,
+	)
+	if err != nil {
+		return explorerStats{}, fmt.Errorf("build freshness snapshot: %w", err)
+	}
+	if len(statsIssues) > 0 {
+		if sampling.Issues == nil {
+			sampling.Issues = map[string]string{}
+		}
+		for key, value := range statsIssues {
+			sampling.Issues[key] = value
+		}
+	}
+	stats.Freshness = snapshot
+	stats.Completeness = completeness
+	stats.Sampling = sampling
+
 	return stats, nil
 }
 

backend/api/rest/stats_internal_test.go

@@ -2,10 +2,17 @@ package rest
 
 import (
 	"context"
+	"database/sql"
+	"encoding/json"
 	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
 	"strings"
 	"testing"
+	"time"
 
+	"github.com/explorer/backend/api/freshness"
 	"github.com/jackc/pgx/v5"
 	"github.com/stretchr/testify/require"
 )
@@ -19,23 +26,56 @@ func (r fakeStatsRow) Scan(dest ...any) error {
 }
 
 func TestLoadExplorerStatsReturnsValues(t *testing.T) {
+	rpc := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var req struct {
+			Method string `json:"method"`
+		}
+		require.NoError(t, json.NewDecoder(r.Body).Decode(&req))
+		w.Header().Set("Content-Type", "application/json")
+		switch req.Method {
+		case "eth_blockNumber":
+			_, _ = w.Write([]byte(`{"jsonrpc":"2.0","id":1,"result":"0x2c"}`))
+		case "eth_getBlockByNumber":
+			ts := time.Now().Add(-2 * time.Second).Unix()
+			_, _ = w.Write([]byte(`{"jsonrpc":"2.0","id":1,"result":{"timestamp":"0x` + strconv.FormatInt(ts, 16) + `"}}`))
+		default:
+			http.Error(w, `{"jsonrpc":"2.0","id":1,"error":{"message":"unsupported"}}`, http.StatusBadRequest)
+		}
+	}))
+	defer rpc.Close()
+	t.Setenv("RPC_URL", rpc.URL)
 	var call int
 	queryRow := func(_ context.Context, _ string, _ ...any) pgx.Row {
 		call++
 		return fakeStatsRow{
 			scan: func(dest ...any) error {
-				target, ok := dest[0].(*int64)
-				require.True(t, ok)
-
 				switch call {
 				case 1:
-					*target = 11
+					*dest[0].(*int64) = 11
 				case 2:
-					*target = 22
+					*dest[0].(*int64) = 22
 				case 3:
-					*target = 33
+					*dest[0].(*int64) = 33
 				case 4:
-					*target = 44
+					*dest[0].(*int64) = 44
+				case 5:
+					*dest[0].(*sql.NullFloat64) = sql.NullFloat64{Float64: 2000, Valid: true}
+				case 6:
+					*dest[0].(*sql.NullFloat64) = sql.NullFloat64{Float64: 1.25, Valid: true}
+				case 7:
+					*dest[0].(*sql.NullFloat64) = sql.NullFloat64{Float64: 37.5, Valid: true}
+				case 8:
+					*dest[0].(*sql.NullInt64) = sql.NullInt64{Int64: 18, Valid: true}
+				case 9:
+					*dest[0].(*int64) = 44
+					*dest[1].(*time.Time) = time.Now().Add(-2 * time.Second)
+				case 10:
+					*dest[0].(*string) = "0xabc"
+					*dest[1].(*int64) = 40
+					*dest[2].(*time.Time) = time.Now().Add(-5 * time.Second)
+				case 11:
+					*dest[0].(*int64) = 40
+					*dest[1].(*time.Time) = time.Now().Add(-5 * time.Second)
 				default:
 					t.Fatalf("unexpected query call %d", call)
 				}
@@ -50,9 +90,25 @@ func TestLoadExplorerStatsReturnsValues(t *testing.T) {
 	require.Equal(t, int64(22), stats.TotalTransactions)
 	require.Equal(t, int64(33), stats.TotalAddresses)
 	require.Equal(t, int64(44), stats.LatestBlock)
+	require.NotNil(t, stats.AverageBlockTime)
+	require.Equal(t, 2000.0, *stats.AverageBlockTime)
+	require.NotNil(t, stats.GasPrices)
+	require.NotNil(t, stats.GasPrices.Average)
+	require.Equal(t, 1.25, *stats.GasPrices.Average)
+	require.NotNil(t, stats.NetworkUtilizationPercentage)
+	require.Equal(t, 37.5, *stats.NetworkUtilizationPercentage)
+	require.NotNil(t, stats.TransactionsToday)
+	require.Equal(t, int64(18), *stats.TransactionsToday)
+	require.NotNil(t, stats.Freshness.ChainHead.BlockNumber)
+	require.Equal(t, int64(40), *stats.Freshness.LatestIndexedTransaction.BlockNumber)
+	require.Equal(t, int64(4), *stats.Freshness.LatestNonEmptyBlock.DistanceFromHead)
+	require.Equal(t, "reported", string(stats.Freshness.ChainHead.Source))
+	require.Equal(t, freshness.CompletenessComplete, stats.Completeness.GasMetrics)
+	require.Equal(t, freshness.CompletenessComplete, stats.Completeness.UtilizationMetric)
 }
 
 func TestLoadExplorerStatsReturnsErrorWhenQueryFails(t *testing.T) {
+	t.Setenv("RPC_URL", "")
 	queryRow := func(_ context.Context, query string, _ ...any) pgx.Row {
 		return fakeStatsRow{
 			scan: func(dest ...any) error {

backend/api/rest/track_routes.go

@@ -1,10 +1,13 @@
 package rest
 
 import (
+	"context"
 	"net/http"
 	"os"
 	"strings"
+	"time"
 
+	"github.com/explorer/backend/api/freshness"
 	"github.com/explorer/backend/api/middleware"
 	"github.com/explorer/backend/api/track1"
 	"github.com/explorer/backend/api/track2"
@@ -47,7 +50,27 @@ func (s *Server) SetupTrackRoutes(mux *http.ServeMux, authMiddleware *middleware
 	}
 
 	rpcGateway := gateway.NewRPCGateway(rpcURL, cache, rateLimiter)
-	track1Server := track1.NewServer(rpcGateway)
+	track1Server := track1.NewServer(rpcGateway, func(ctx context.Context) (*freshness.Snapshot, *freshness.SummaryCompleteness, *freshness.Sampling, error) {
+		if s.db == nil {
+			return nil, nil, nil, nil
+		}
+		now := time.Now().UTC()
+		snapshot, completeness, sampling, err := freshness.BuildSnapshot(
+			ctx,
+			s.chainID,
+			s.db.QueryRow,
+			func(ctx context.Context) (*freshness.Reference, error) {
+				return freshness.ProbeChainHead(ctx, rpcURL)
+			},
+			now,
+			nil,
+			nil,
+		)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		return &snapshot, &completeness, &sampling, nil
+	})
 
 	// Track 1 routes (public, optional auth)
 	mux.HandleFunc("/api/v1/track1/blocks/latest", track1Server.HandleLatestBlocks)