demo test + Peer

daemons/node/indexer/native.go

@@ -4,7 +4,10 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"math/rand"
+	"slices"
+	"strings"
 	"sync"
 	"time"
 
@@ -12,19 +15,24 @@ import (
 
 	oclib "cloud.o-forge.io/core/oc-lib"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
-	"github.com/libp2p/go-libp2p/core/host"
 	"github.com/libp2p/go-libp2p/core/network"
 	pp "github.com/libp2p/go-libp2p/core/peer"
 )
 
 const (
-	// IndexerTTL is 10% above the recommended 60s heartbeat interval.
-	IndexerTTL = 66 * time.Second
+	// IndexerTTL is the lifetime of a live-indexer cache entry. Set to 50% above
+	// the recommended 60s heartbeat interval so a single delayed renewal does not
+	// evict a healthy indexer from the native's cache.
+	IndexerTTL = 90 * time.Second
 	// offloadInterval is how often the native checks if it can release responsible peers.
 	offloadInterval = 30 * time.Second
 	// dhtRefreshInterval is how often the background goroutine queries the DHT for
 	// known-but-expired indexer entries (written by neighbouring natives).
 	dhtRefreshInterval = 30 * time.Second
+	// maxFallbackPeers caps how many peers the native will accept in self-delegation
+	// mode. Beyond this limit the native refuses to act as a fallback indexer so it
+	// is not overwhelmed during prolonged indexer outages.
+	maxFallbackPeers = 50
 )
 
 // liveIndexerEntry tracks a registered indexer in the native's in-memory cache and DHT.
@@ -43,7 +51,7 @@ type NativeState struct {
 	// knownPeerIDs accumulates all indexer PeerIDs ever seen (local stream or gossip).
 	// Used by refreshIndexersFromDHT to re-hydrate expired entries from the shared DHT,
 	// including entries written by other natives.
-	knownPeerIDs map[string]struct{}
+	knownPeerIDs map[string]string
 	knownMu      sync.RWMutex
 }
 
@@ -51,7 +59,7 @@ func newNativeState() *NativeState {
 	return &NativeState{
 		liveIndexers:     map[string]*liveIndexerEntry{},
 		responsiblePeers: map[pp.ID]struct{}{},
-		knownPeerIDs:     map[string]struct{}{},
+		knownPeerIDs:     map[string]string{},
 	}
 }
 
@@ -92,10 +100,12 @@ func (v IndexerRecordValidator) Select(_ string, values [][]byte) (int, error) {
 // Must be called after DHT is initialized.
 func (ix *IndexerService) InitNative() {
 	ix.Native = newNativeState()
-	ix.Host.SetStreamHandler(common.ProtocolIndexerHeartbeat, ix.HandleNodeHeartbeat) // specific heartbeat for Indexer.
+	ix.Host.SetStreamHandler(common.ProtocolHeartbeat, ix.HandleHeartbeat) // specific heartbeat for Indexer.
 	ix.Host.SetStreamHandler(common.ProtocolNativeSubscription, ix.handleNativeSubscription)
 	ix.Host.SetStreamHandler(common.ProtocolNativeGetIndexers, ix.handleNativeGetIndexers)
 	ix.Host.SetStreamHandler(common.ProtocolNativeConsensus, ix.handleNativeConsensus)
+	ix.Host.SetStreamHandler(common.ProtocolNativeGetPeers, ix.handleNativeGetPeers)
+	ix.Host.SetStreamHandler(common.ProtocolIndexerGetNatives, ix.handleGetNatives)
 	ix.subscribeIndexerRegistry()
 	// Ensure long connections to other configured natives (native-to-native mesh).
 	common.EnsureNativePeers(ix.Host)
@@ -107,8 +117,15 @@ func (ix *IndexerService) InitNative() {
 // registered indexer PeerIDs to one another, enabling cross-native DHT discovery.
 func (ix *IndexerService) subscribeIndexerRegistry() {
 	logger := oclib.GetLogger()
-	ix.PS.RegisterTopicValidator(common.TopicIndexerRegistry, func(_ context.Context, _ pp.ID, _ *pubsub.Message) bool {
-		return true
+	ix.PS.RegisterTopicValidator(common.TopicIndexerRegistry, func(_ context.Context, _ pp.ID, msg *pubsub.Message) bool {
+		// Reject empty or syntactically invalid multiaddrs before they reach the
+		// message loop. A compromised native could otherwise gossip arbitrary data.
+		addr := string(msg.Data)
+		if addr == "" {
+			return false
+		}
+		_, err := pp.AddrInfoFromString(addr)
+		return err == nil
 	})
 	topic, err := ix.PS.Join(common.TopicIndexerRegistry)
 	if err != nil {
@@ -130,29 +147,38 @@ func (ix *IndexerService) subscribeIndexerRegistry() {
 			if err != nil {
 				return
 			}
-			peerID := string(msg.Data)
-			if peerID == "" {
+			addr := string(msg.Data)
+			if addr == "" {
 				continue
 			}
+			if peer, err := pp.AddrInfoFromString(addr); err == nil {
+				ix.Native.knownMu.Lock()
+				ix.Native.knownPeerIDs[peer.ID.String()] = addr
+				ix.Native.knownMu.Unlock()
+
+			}
 			// A neighbouring native registered this PeerID; add to known set for DHT refresh.
-			ix.Native.knownMu.Lock()
-			ix.Native.knownPeerIDs[peerID] = struct{}{}
-			ix.Native.knownMu.Unlock()
+
 		}
 	}()
 }
 
-// handleNativeSubscription stores an indexer's alive registration in the DHT cache.
+// handleNativeSubscription stores an indexer's alive registration in the local cache
+// immediately, then persists it to the DHT asynchronously.
 // The stream is temporary: indexer sends one IndexerRegistration and closes.
 func (ix *IndexerService) handleNativeSubscription(s network.Stream) {
 	defer s.Close()
 	logger := oclib.GetLogger()
 
+	logger.Info().Msg("Subscription")
+
 	var reg common.IndexerRegistration
 	if err := json.NewDecoder(s).Decode(&reg); err != nil {
 		logger.Err(err).Msg("native subscription: decode")
 		return
 	}
+	logger.Info().Msg("Subscription " + reg.Addr)
+
 	if reg.Addr == "" {
 		logger.Error().Msg("native subscription: missing addr")
 		return
@@ -166,30 +192,23 @@ func (ix *IndexerService) handleNativeSubscription(s network.Stream) {
 		reg.PeerID = ad.ID.String()
 	}
 
-	expiry := time.Now().UTC().Add(IndexerTTL)
+	// Build entry with a fresh TTL — must happen before the cache write so the 66s
+	// window is not consumed by DHT retries.
 	entry := &liveIndexerEntry{
 		PeerID:    reg.PeerID,
 		Addr:      reg.Addr,
-		ExpiresAt: expiry,
+		ExpiresAt: time.Now().UTC().Add(IndexerTTL),
 	}
 
-	// Persist in DHT with 66s TTL.
-	key := ix.genIndexerKey(reg.PeerID)
-	if data, err := json.Marshal(entry); err == nil {
-		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-		if err := ix.DHT.PutValue(ctx, key, data); err != nil {
-			logger.Err(err).Msg("native subscription: DHT put")
-		}
-		cancel()
-	}
-
-	// Update local cache and known set.
+	// Update local cache and known set immediately so concurrent GetIndexers calls
+	// can already see this indexer without waiting for the DHT write to complete.
 	ix.Native.liveIndexersMu.Lock()
+	_, isRenewal := ix.Native.liveIndexers[reg.PeerID]
 	ix.Native.liveIndexers[reg.PeerID] = entry
 	ix.Native.liveIndexersMu.Unlock()
 
 	ix.Native.knownMu.Lock()
-	ix.Native.knownPeerIDs[reg.PeerID] = struct{}{}
+	ix.Native.knownPeerIDs[reg.PeerID] = reg.Addr
 	ix.Native.knownMu.Unlock()
 
 	// Gossip PeerID to neighbouring natives so they discover it via DHT.
@@ -197,16 +216,46 @@ func (ix *IndexerService) handleNativeSubscription(s network.Stream) {
 	topic := ix.LongLivedPubSubs[common.TopicIndexerRegistry]
 	ix.PubsubMu.RUnlock()
 	if topic != nil {
-		if err := topic.Publish(context.Background(), []byte(reg.PeerID)); err != nil {
+		if err := topic.Publish(context.Background(), []byte(reg.Addr)); err != nil {
 			logger.Err(err).Msg("native subscription: registry gossip publish")
 		}
 	}
 
-	logger.Info().Str("peer", reg.PeerID).Msg("native: indexer registered")
+	if isRenewal {
+		logger.Debug().Str("peer", reg.PeerID).Msg("native: indexer TTL renewed : " + fmt.Sprintf("%v", len(ix.Native.liveIndexers)))
+	} else {
+		logger.Info().Str("peer", reg.PeerID).Msg("native: indexer registered : " + fmt.Sprintf("%v", len(ix.Native.liveIndexers)))
+	}
+
+	// Persist in DHT asynchronously — retries must not block the handler or consume
+	// the local cache TTL.
+	key := ix.genIndexerKey(reg.PeerID)
+	data, err := json.Marshal(entry)
+	if err != nil {
+		logger.Err(err).Msg("native subscription: marshal entry")
+		return
+	}
+	go func() {
+		for {
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			if err := ix.DHT.PutValue(ctx, key, data); err != nil {
+				cancel()
+				logger.Err(err).Msg("native subscription: DHT put " + key)
+				if strings.Contains(err.Error(), "failed to find any peer in table") {
+					time.Sleep(10 * time.Second)
+					continue
+				}
+				return
+			}
+			cancel()
+			return
+		}
+	}()
 }
 
 // handleNativeGetIndexers returns this native's own list of reachable indexers.
-// If none are available, it self-delegates (becomes the fallback indexer for the caller).
+// Self-delegation (native acting as temporary fallback indexer) is only permitted
+// for nodes — never for peers that are themselves registered indexers in knownPeerIDs.
 // The consensus across natives is the responsibility of the requesting node/indexer.
 func (ix *IndexerService) handleNativeGetIndexers(s network.Stream) {
 	defer s.Close()
@@ -220,14 +269,20 @@ func (ix *IndexerService) handleNativeGetIndexers(s network.Stream) {
 	if req.Count <= 0 {
 		req.Count = 3
 	}
-
-	reachable := ix.reachableLiveIndexers()
+	callerPeerID := s.Conn().RemotePeer().String()
+	reachable := ix.reachableLiveIndexers(req.Count, callerPeerID)
 	var resp common.GetIndexersResponse
 
 	if len(reachable) == 0 {
-		// No indexers known: become temporary fallback for this caller.
-		ix.selfDelegate(s.Conn().RemotePeer(), &resp)
-		logger.Info().Str("peer", s.Conn().RemotePeer().String()).Msg("native: no indexers, acting as fallback")
+		// No live indexers reachable — try to self-delegate.
+		if ix.selfDelegate(s.Conn().RemotePeer(), &resp) {
+			logger.Info().Str("peer", callerPeerID).Msg("native: no indexers, acting as fallback for node")
+		} else {
+			// Fallback pool saturated: return empty so the caller retries another
+			// native instead of piling more load onto this one.
+			logger.Warn().Str("peer", callerPeerID).Int("pool", maxFallbackPeers).Msg(
+				"native: fallback pool saturated, refusing self-delegation")
+		}
 	} else {
 		rand.Shuffle(len(reachable), func(i, j int) { reachable[i], reachable[j] = reachable[j], reachable[i] })
 		if req.Count > len(reachable) {
@@ -255,7 +310,7 @@ func (ix *IndexerService) handleNativeConsensus(s network.Stream) {
 		return
 	}
 
-	myList := ix.reachableLiveIndexers()
+	myList := ix.reachableLiveIndexers(-1, s.Conn().RemotePeer().String())
 	mySet := make(map[string]struct{}, len(myList))
 	for _, addr := range myList {
 		mySet[addr] = struct{}{}
@@ -285,31 +340,56 @@ func (ix *IndexerService) handleNativeConsensus(s network.Stream) {
 }
 
 // selfDelegate marks the caller as a responsible peer and exposes this native's own
-// address as its temporary indexer.
-func (ix *IndexerService) selfDelegate(remotePeer pp.ID, resp *common.GetIndexersResponse) {
+// address as its temporary indexer. Returns false when the fallback pool is saturated
+// (maxFallbackPeers reached) — the caller must return an empty response so the node
+// retries later instead of pinning indefinitely to an overloaded native.
+func (ix *IndexerService) selfDelegate(remotePeer pp.ID, resp *common.GetIndexersResponse) bool {
 	ix.Native.responsibleMu.Lock()
-	ix.Native.responsiblePeers[remotePeer] = struct{}{}
-	ix.Native.responsibleMu.Unlock()
-	resp.IsSelfFallback = true
-	for _, a := range ix.Host.Addrs() {
-		resp.Indexers = []string{a.String() + "/p2p/" + ix.Host.ID().String()}
-		break
+	defer ix.Native.responsibleMu.Unlock()
+	if len(ix.Native.responsiblePeers) >= maxFallbackPeers {
+		return false
 	}
+	ix.Native.responsiblePeers[remotePeer] = struct{}{}
+	resp.IsSelfFallback = true
+	resp.Indexers = []string{ix.Host.Addrs()[len(ix.Host.Addrs())-1].String() + "/p2p/" + ix.Host.ID().String()}
+	return true
 }
 
 // reachableLiveIndexers returns the multiaddrs of non-expired, pingable indexers
 // from the local cache (kept fresh by refreshIndexersFromDHT in background).
-func (ix *IndexerService) reachableLiveIndexers() []string {
+func (ix *IndexerService) reachableLiveIndexers(count int, from ...string) []string {
 	ix.Native.liveIndexersMu.RLock()
 	now := time.Now().UTC()
 	candidates := []*liveIndexerEntry{}
 	for _, e := range ix.Native.liveIndexers {
-		if e.ExpiresAt.After(now) {
+		fmt.Println("liveIndexers", slices.Contains(from, e.PeerID), from, e.PeerID)
+		if e.ExpiresAt.After(now) && !slices.Contains(from, e.PeerID) {
 			candidates = append(candidates, e)
 		}
 	}
 	ix.Native.liveIndexersMu.RUnlock()
 
+	fmt.Println("midway...", candidates, from, ix.Native.knownPeerIDs)
+
+	if (count > 0 && len(candidates) < count) || count < 0 {
+		ix.Native.knownMu.RLock()
+		for k, v := range ix.Native.knownPeerIDs {
+			// Include peers whose liveIndexers entry is absent OR expired.
+			// A non-nil but expired entry means the peer was once known but
+			// has since timed out — PeerIsAlive below will decide if it's back.
+			fmt.Println("knownPeerIDs", slices.Contains(from, k), from, k)
+			if !slices.Contains(from, k) {
+				candidates = append(candidates, &liveIndexerEntry{
+					PeerID: k,
+					Addr:   v,
+				})
+			}
+		}
+		ix.Native.knownMu.RUnlock()
+	}
+
+	fmt.Println("midway...1", candidates)
+
 	reachable := []string{}
 	for _, e := range candidates {
 		ad, err := pp.AddrInfoFromString(e.Addr)
@@ -371,6 +451,12 @@ func (ix *IndexerService) refreshIndexersFromDHT() {
 				ix.Native.liveIndexers[best.PeerID] = best
 				ix.Native.liveIndexersMu.Unlock()
 				logger.Info().Str("peer", best.PeerID).Msg("native: refreshed indexer from DHT")
+			} else {
+				// DHT has no fresh entry — peer is gone, prune from known set.
+				ix.Native.knownMu.Lock()
+				delete(ix.Native.knownPeerIDs, pid)
+				ix.Native.knownMu.Unlock()
+				logger.Info().Str("peer", pid).Msg("native: pruned stale peer from knownPeerIDs")
 			}
 		}
 	}
@@ -387,30 +473,107 @@ func (ix *IndexerService) runOffloadLoop() {
 	defer t.Stop()
 	logger := oclib.GetLogger()
 	for range t.C {
+		fmt.Println("runOffloadLoop", ix.Native.responsiblePeers)
 		ix.Native.responsibleMu.RLock()
 		count := len(ix.Native.responsiblePeers)
 		ix.Native.responsibleMu.RUnlock()
 		if count == 0 {
 			continue
 		}
-		if len(ix.reachableLiveIndexers()) > 0 {
-			ix.Native.responsibleMu.Lock()
-			ix.Native.responsiblePeers = map[pp.ID]struct{}{}
-			ix.Native.responsibleMu.Unlock()
+		ix.Native.responsibleMu.RLock()
+		peerIDS := []string{}
+		for p := range ix.Native.responsiblePeers {
+			peerIDS = append(peerIDS, p.String())
+		}
+		fmt.Println("COUNT --> ", count, len(ix.reachableLiveIndexers(-1, peerIDS...)))
+		ix.Native.responsibleMu.RUnlock()
+		if len(ix.reachableLiveIndexers(-1, peerIDS...)) > 0 {
+			ix.Native.responsibleMu.RLock()
+			released := ix.Native.responsiblePeers
+			ix.Native.responsibleMu.RUnlock()
+
+			// Reset (not Close) heartbeat streams of released peers.
+			// Close() only half-closes the native's write direction — the peer's write
+			// direction stays open and sendHeartbeat never sees an error.
+			// Reset() abruptly terminates both directions, making the peer's next
+			// json.Encode return an error which triggers replenishIndexersFromNative.
+			ix.StreamMU.Lock()
+			if streams := ix.StreamRecords[common.ProtocolHeartbeat]; streams != nil {
+				for pid := range released {
+					if rec, ok := streams[pid]; ok {
+						if rec.HeartbeatStream != nil && rec.HeartbeatStream.Stream != nil {
+							rec.HeartbeatStream.Stream.Reset()
+						}
+						ix.Native.responsibleMu.Lock()
+						delete(ix.Native.responsiblePeers, pid)
+						ix.Native.responsibleMu.Unlock()
+
+						delete(streams, pid)
+						logger.Info().Str("peer", pid.String()).Str("proto", string(common.ProtocolHeartbeat)).Msg(
+							"native: offload — stream reset, peer will reconnect to real indexer")
+					} else {
+						// No recorded heartbeat stream for this peer: either it never
+						// passed the score check (new peer, uptime=0 → score<75) or the
+						// stream was GC'd.  We cannot send a Reset signal, so close the
+						// whole connection instead — this makes the peer's sendHeartbeat
+						// return an error, which triggers replenishIndexersFromNative and
+						// migrates it to a real indexer.
+						ix.Native.responsibleMu.Lock()
+						delete(ix.Native.responsiblePeers, pid)
+						ix.Native.responsibleMu.Unlock()
+						go ix.Host.Network().ClosePeer(pid)
+						logger.Info().Str("peer", pid.String()).Msg(
+							"native: offload — no heartbeat stream, closing connection so peer re-requests real indexers")
+					}
+				}
+
+			}
+			ix.StreamMU.Unlock()
+
 			logger.Info().Int("released", count).Msg("native: offloaded responsible peers to real indexers")
 		}
 	}
 }
 
+// handleNativeGetPeers returns a random selection of this native's known native
+// contacts, excluding any in the request's Exclude list.
+func (ix *IndexerService) handleNativeGetPeers(s network.Stream) {
+	defer s.Close()
+	logger := oclib.GetLogger()
+
+	var req common.GetNativePeersRequest
+	if err := json.NewDecoder(s).Decode(&req); err != nil {
+		logger.Err(err).Msg("native get peers: decode")
+		return
+	}
+	if req.Count <= 0 {
+		req.Count = 1
+	}
+
+	excludeSet := make(map[string]struct{}, len(req.Exclude))
+	for _, e := range req.Exclude {
+		excludeSet[e] = struct{}{}
+	}
+
+	common.StreamNativeMu.RLock()
+	candidates := make([]string, 0, len(common.StaticNatives))
+	for addr := range common.StaticNatives {
+		if _, excluded := excludeSet[addr]; !excluded {
+			candidates = append(candidates, addr)
+		}
+	}
+	common.StreamNativeMu.RUnlock()
+
+	rand.Shuffle(len(candidates), func(i, j int) { candidates[i], candidates[j] = candidates[j], candidates[i] })
+	if req.Count > len(candidates) {
+		req.Count = len(candidates)
+	}
+
+	resp := common.GetNativePeersResponse{Peers: candidates[:req.Count]}
+	if err := json.NewEncoder(s).Encode(resp); err != nil {
+		logger.Err(err).Msg("native get peers: encode response")
+	}
+}
+
 // StartNativeRegistration starts a goroutine that periodically registers this
 // indexer with all configured native indexers (every RecommendedHeartbeatInterval).
-func StartNativeRegistration(h host.Host, nativeAddressesStr string) {
-	go func() {
-		common.RegisterWithNative(h, nativeAddressesStr)
-		t := time.NewTicker(common.RecommendedHeartbeatInterval)
-		defer t.Stop()
-		for range t.C {
-			common.RegisterWithNative(h, nativeAddressesStr)
-		}
-	}()
-}