demo test + Peer

2026-03-03 16:38:24 +01:00
parent 79aa3cc2b3
commit ef3d998ead
41 changed files with 2360 additions and 578 deletions
--- a/daemons/node/common/common_stream.go
+++ b/daemons/node/common/common_stream.go
@@ -1,7 +1,6 @@
 package common

 import (
-	"bytes"
 	"context"
 	cr "crypto/rand"
 	"encoding/json"
@@ -28,6 +27,12 @@ type LongLivedStreamRecordedService[T interface{}] struct {
 	StreamRecords map[protocol.ID]map[pp.ID]*StreamRecord[T]
 	StreamMU      sync.RWMutex
 	maxNodesConn  int
+	// AfterHeartbeat is an optional hook called after each successful heartbeat update.
+	// The indexer sets it to republish the embedded signed record to the DHT.
+	AfterHeartbeat func(pid pp.ID)
+	// AfterDelete is called after gc() evicts an expired peer, outside the lock.
+	// name and did may be empty if the HeartbeatStream had no metadata.
+	AfterDelete func(pid pp.ID, name string, did string)
 }

 func NewStreamRecordedService[T interface{}](h host.Host, maxNodesConn int) *LongLivedStreamRecordedService[T] {
@@ -54,16 +59,29 @@ func (ix *LongLivedStreamRecordedService[T]) StartGC(interval time.Duration) {

 func (ix *LongLivedStreamRecordedService[T]) gc() {
 	ix.StreamMU.Lock()
-	defer ix.StreamMU.Unlock()
 	now := time.Now().UTC()
 	if ix.StreamRecords[ProtocolHeartbeat] == nil {
 		ix.StreamRecords[ProtocolHeartbeat] = map[pp.ID]*StreamRecord[T]{}
+		ix.StreamMU.Unlock()
 		return
 	}
 	streams := ix.StreamRecords[ProtocolHeartbeat]
+	fmt.Println(StaticNatives, StaticIndexers, streams)

+	type gcEntry struct {
+		pid  pp.ID
+		name string
+		did  string
+	}
+	var evicted []gcEntry
 	for pid, rec := range streams {
 		if now.After(rec.HeartbeatStream.Expiry) || now.Sub(rec.HeartbeatStream.UptimeTracker.LastSeen) > 2*rec.HeartbeatStream.Expiry.Sub(now) {
+			name, did := "", ""
+			if rec.HeartbeatStream != nil {
+				name = rec.HeartbeatStream.Name
+				did = rec.HeartbeatStream.DID
+			}
+			evicted = append(evicted, gcEntry{pid, name, did})
 			for _, sstreams := range ix.StreamRecords {
 				if sstreams[pid] != nil {
 					delete(sstreams, pid)
@@ -71,6 +89,13 @@ func (ix *LongLivedStreamRecordedService[T]) gc() {
 			}
 		}
 	}
+	ix.StreamMU.Unlock()
+
+	if ix.AfterDelete != nil {
+		for _, e := range evicted {
+			ix.AfterDelete(e.pid, e.name, e.did)
+		}
+	}
 }

 func (ix *LongLivedStreamRecordedService[T]) Snapshot(interval time.Duration) {
@@ -101,8 +126,10 @@ func (ix *LongLivedStreamRecordedService[T]) snapshot() []*StreamRecord[T] {
 	return out
 }

-func (ix *LongLivedStreamRecordedService[T]) HandleNodeHeartbeat(s network.Stream) {
+func (ix *LongLivedStreamRecordedService[T]) HandleHeartbeat(s network.Stream) {
+	logger := oclib.GetLogger()
 	defer s.Close()
+	dec := json.NewDecoder(s)
 	for {
 		ix.StreamMU.Lock()
 		if ix.StreamRecords[ProtocolHeartbeat] == nil {
@@ -114,17 +141,37 @@ func (ix *LongLivedStreamRecordedService[T]) HandleNodeHeartbeat(s network.Strea
 			streamsAnonym[k] = v
 		}
 		ix.StreamMU.Unlock()
-
-		pid, hb, err := CheckHeartbeat(ix.Host, s, streamsAnonym, &ix.StreamMU, ix.maxNodesConn)
+		pid, hb, err := CheckHeartbeat(ix.Host, s, dec, streamsAnonym, &ix.StreamMU, ix.maxNodesConn)
 		if err != nil {
+			// Stream-level errors (EOF, reset, closed) mean the connection is gone
+			// — exit so the goroutine doesn't spin forever on a dead stream.
+			// Metric/policy errors (score too low, too many connections) are transient
+			// — those are also stream-terminal since the stream carries one session.
+			if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) ||
+				strings.Contains(err.Error(), "reset") ||
+				strings.Contains(err.Error(), "closed") ||
+				strings.Contains(err.Error(), "too many connections") {
+				logger.Info().Err(err).Msg("heartbeat stream terminated, closing handler")
+				return
+			}
+			logger.Warn().Err(err).Msg("heartbeat check failed, retrying on same stream")
 			continue
 		}
 		ix.StreamMU.Lock()
 		// if record already seen update last seen
 		if rec, ok := streams[*pid]; ok {
 			rec.DID = hb.DID
+			if rec.HeartbeatStream == nil {
+				rec.HeartbeatStream = hb.Stream
+			}
 			rec.HeartbeatStream = hb.Stream
-			rec.HeartbeatStream.UptimeTracker.LastSeen = time.Now().UTC()
+			if rec.HeartbeatStream.UptimeTracker == nil {
+				rec.HeartbeatStream.UptimeTracker = &UptimeTracker{
+					FirstSeen: time.Now().UTC(),
+					LastSeen:  time.Now().UTC(),
+				}
+			}
+			logger.Info().Msg("A new node is updated : " + pid.String())
 		} else {
 			hb.Stream.UptimeTracker = &UptimeTracker{
 				FirstSeen: time.Now().UTC(),
@@ -134,37 +181,51 @@ func (ix *LongLivedStreamRecordedService[T]) HandleNodeHeartbeat(s network.Strea
 				DID:             hb.DID,
 				HeartbeatStream: hb.Stream,
 			}
+			logger.Info().Msg("A new node is subscribed : " + pid.String())
 		}
 		ix.StreamMU.Unlock()
+		// Let the indexer republish the embedded signed record to the DHT.
+		if ix.AfterHeartbeat != nil {
+			ix.AfterHeartbeat(*pid)
+		}
 	}
 }

-func CheckHeartbeat(h host.Host, s network.Stream, streams map[pp.ID]HeartBeatStreamed, lock *sync.RWMutex, maxNodes int) (*pp.ID, *Heartbeat, error) {
+func CheckHeartbeat(h host.Host, s network.Stream, dec *json.Decoder, streams map[pp.ID]HeartBeatStreamed, lock *sync.RWMutex, maxNodes int) (*pp.ID, *Heartbeat, error) {
 	if len(h.Network().Peers()) >= maxNodes {
 		return nil, nil, fmt.Errorf("too many connections, try another indexer")
 	}
 	var hb Heartbeat
-	if err := json.NewDecoder(s).Decode(&hb); err != nil {
+	if err := dec.Decode(&hb); err != nil {
 		return nil, nil, err
 	}
-	if ok, bpms, err := getBandwidthChallengeRate(MinPayloadChallenge+int(rand.Float64()*(MaxPayloadChallenge-MinPayloadChallenge)), s); err != nil {
-		return nil, nil, err
-	} else if !ok {
-		return nil, nil, fmt.Errorf("Not a proper peer")
-	} else {
+	_, bpms, _ := getBandwidthChallengeRate(h, s.Conn().RemotePeer(), MinPayloadChallenge+int(rand.Float64()*(MaxPayloadChallenge-MinPayloadChallenge)))
+	{
 		pid, err := pp.Decode(hb.PeerID)
 		if err != nil {
 			return nil, nil, err
 		}
 		upTime := float64(0)
+		isFirstHeartbeat := true
 		lock.Lock()
 		if rec, ok := streams[pid]; ok && rec.GetUptimeTracker() != nil {
 			upTime = rec.GetUptimeTracker().Uptime().Hours() / float64(time.Since(TimeWatcher).Hours())
+			isFirstHeartbeat = false
 		}
 		lock.Unlock()
 		diversity := getDiversityRate(h, hb.IndexersBinded)
+		fmt.Println(upTime, bpms, diversity)
 		hb.ComputeIndexerScore(upTime, bpms, diversity)
-		if hb.Score < 75 {
+		// First heartbeat: uptime is always 0 so the score ceiling is 60, below the
+		// steady-state threshold of 75. Use a lower admission threshold so new peers
+		// can enter and start accumulating uptime. Subsequent heartbeats must meet
+		// the full threshold once uptime is tracked.
+		minScore := float64(50)
+		if isFirstHeartbeat {
+			minScore = 40
+		}
+		fmt.Println(hb.Score, minScore)
+		if hb.Score < minScore {
 			return nil, nil, errors.New("not enough trusting value")
 		}
 		hb.Stream = &Stream{
@@ -178,11 +239,13 @@ func CheckHeartbeat(h host.Host, s network.Stream, streams map[pp.ID]HeartBeatSt
 }

 func getDiversityRate(h host.Host, peers []string) float64 {
+
 	peers, _ = checkPeers(h, peers)
 	diverse := []string{}
 	for _, p := range peers {
 		ip, err := ExtractIP(p)
 		if err != nil {
+			fmt.Println("NO IP", p, err)
 			continue
 		}
 		div := ip.Mask(net.CIDRMask(24, 32)).String()
@@ -190,6 +253,9 @@ func getDiversityRate(h host.Host, peers []string) float64 {
 			diverse = append(diverse, div)
 		}
 	}
+	if len(diverse) == 0 || len(peers) == 0 {
+		return 1
+	}
 	return float64(len(diverse) / len(peers))
 }

@@ -211,35 +277,42 @@ func checkPeers(h host.Host, peers []string) ([]string, []string) {
 	return concretePeer, ips
 }

-const MaxExpectedMbps = 50.0
+const MaxExpectedMbps = 100.0
 const MinPayloadChallenge = 512
 const MaxPayloadChallenge = 2048
 const BaseRoundTrip = 400 * time.Millisecond

-func getBandwidthChallengeRate(payloadSize int, s network.Stream) (bool, float64, error) {
-	// Génération payload aléatoire
+// getBandwidthChallengeRate opens a dedicated ProtocolBandwidthProbe stream to
+// remotePeer, sends a random payload, reads the echo, and computes throughput.
+// Using a separate stream avoids mixing binary data on the JSON heartbeat stream
+// and ensures the echo handler is actually running on the remote side.
+func getBandwidthChallengeRate(h host.Host, remotePeer pp.ID, payloadSize int) (bool, float64, error) {
 	payload := make([]byte, payloadSize)
-	_, err := cr.Read(payload)
+	if _, err := cr.Read(payload); err != nil {
+		return false, 0, err
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	s, err := h.NewStream(ctx, remotePeer, ProtocolBandwidthProbe)
 	if err != nil {
 		return false, 0, err
 	}
+	defer s.Reset()
+	s.SetDeadline(time.Now().Add(10 * time.Second))
 	start := time.Now()
-	// send on heartbeat stream the challenge
 	if _, err = s.Write(payload); err != nil {
 		return false, 0, err
 	}
-	// read back
+	s.CloseWrite()
+	// Half-close the write side so the handler's io.Copy sees EOF and stops.
+	// Read the echo.
 	response := make([]byte, payloadSize)
-	_, err = io.ReadFull(s, response)
-	if err != nil {
+	if _, err = io.ReadFull(s, response); err != nil {
 		return false, 0, err
 	}

 	duration := time.Since(start)
-	// Verify content
-	if !bytes.Equal(payload, response) {
-		return false, 0, nil // pb or a sadge peer.
-	}
 	maxRoundTrip := BaseRoundTrip + (time.Duration(payloadSize) * (100 * time.Millisecond))
 	mbps := float64(payloadSize*8) / duration.Seconds() / 1e6
 	if duration > maxRoundTrip || mbps < 5.0 {
@@ -345,13 +418,36 @@ var StaticIndexers map[string]*pp.AddrInfo = map[string]*pp.AddrInfo{}
 var StreamMuIndexes sync.RWMutex
 var StreamIndexers ProtocolStream = ProtocolStream{}

-func ConnectToIndexers(h host.Host, minIndexer int, maxIndexer int, myPID pp.ID) error {
+// indexerHeartbeatNudge allows replenishIndexersFromNative to trigger an immediate
+// heartbeat tick after adding new entries to StaticIndexers, without waiting up
+// to 20s for the regular ticker. Buffered(1) so the sender never blocks.
+var indexerHeartbeatNudge = make(chan struct{}, 1)
+
+// NudgeIndexerHeartbeat signals the indexer heartbeat goroutine to fire immediately.
+func NudgeIndexerHeartbeat() {
+	select {
+	case indexerHeartbeatNudge <- struct{}{}:
+	default: // nudge already pending, skip
+	}
+}
+
+func ConnectToIndexers(h host.Host, minIndexer int, maxIndexer int, myPID pp.ID, recordFn ...func() json.RawMessage) error {
 	TimeWatcher = time.Now().UTC()
 	logger := oclib.GetLogger()

-	// If native addresses are configured, bypass static indexer addresses
+	// If native addresses are configured, get the indexer pool from the native mesh,
+	// then start the long-lived heartbeat goroutine toward those indexers.
 	if conf.GetConfig().NativeIndexerAddresses != "" {
-		return ConnectToNatives(h, minIndexer, maxIndexer, myPID)
+		if err := ConnectToNatives(h, minIndexer, maxIndexer, myPID); err != nil {
+			return err
+		}
+		// Step 2: start the long-lived heartbeat goroutine toward the indexer pool.
+		// replaceStaticIndexers/replenishIndexersFromNative update the map in-place
+		// so this single goroutine follows all pool changes automatically.
+		logger.Info().Msg("[native] step 2 — starting long-lived heartbeat to indexer pool")
+		SendHeartbeat(context.Background(), ProtocolHeartbeat, conf.GetConfig().Name,
+			h, StreamIndexers, StaticIndexers, &StreamMuIndexes, 20*time.Second, recordFn...)
+		return nil
 	}

 	addresses := strings.Split(conf.GetConfig().IndexerAddresses, ",")
@@ -360,8 +456,8 @@ func ConnectToIndexers(h host.Host, minIndexer int, maxIndexer int, myPID pp.ID)
 		addresses = addresses[0:maxIndexer]
 	}

+	StreamMuIndexes.Lock()
 	for _, indexerAddr := range addresses {
-		fmt.Println("GENERATE ADDR", indexerAddr)
 		ad, err := pp.AddrInfoFromString(indexerAddr)
 		if err != nil {
 			logger.Err(err)
@@ -369,15 +465,18 @@ func ConnectToIndexers(h host.Host, minIndexer int, maxIndexer int, myPID pp.ID)
 		}
 		StaticIndexers[indexerAddr] = ad
 	}
+	indexerCount := len(StaticIndexers)
+	StreamMuIndexes.Unlock()

-	SendHeartbeat(context.Background(), ProtocolHeartbeat, conf.GetConfig().Name, h, StreamIndexers, StaticIndexers, 20*time.Second) // your indexer is just like a node for the next indexer.
-	if len(StaticIndexers) < minIndexer {
+	SendHeartbeat(context.Background(), ProtocolHeartbeat, conf.GetConfig().Name, h, StreamIndexers, StaticIndexers, &StreamMuIndexes, 20*time.Second, recordFn...) // your indexer is just like a node for the next indexer.
+	if indexerCount < minIndexer {
 		return errors.New("you run a node without indexers... your gonna be isolated.")
 	}
 	return nil
 }

 func AddStreamProtocol(ctx *context.Context, protoS ProtocolStream, h host.Host, proto protocol.ID, id pp.ID, mypid pp.ID, force bool, onStreamCreated *func(network.Stream)) ProtocolStream {
+	logger := oclib.GetLogger()
 	if onStreamCreated == nil {
 		f := func(s network.Stream) {
 			protoS[proto][id] = &Stream{
@@ -400,7 +499,7 @@ func AddStreamProtocol(ctx *context.Context, protoS ProtocolStream, h host.Host,
 		if protoS[proto][id] != nil {
 			protoS[proto][id].Expiry = time.Now().Add(2 * time.Minute)
 		} else {
-			fmt.Println("NEW STREAM", proto, id)
+			logger.Info().Msg("NEW STREAM Generated" + fmt.Sprintf("%v", proto) + " " + id.String())
 			s, err := h.NewStream(*ctx, id, proto)
 			if err != nil {
 				panic(err.Error())
@@ -419,12 +518,16 @@ type Heartbeat struct {
 	Timestamp      int64    `json:"timestamp"`
 	IndexersBinded []string `json:"indexers_binded"`
 	Score          float64
+	// Record carries a fresh signed PeerRecord (JSON) so the receiving indexer
+	// can republish it to the DHT without an extra round-trip.
+	// Only set by nodes (not indexers heartbeating other indexers).
+	Record json.RawMessage `json:"record,omitempty"`
 }

 func (hb *Heartbeat) ComputeIndexerScore(uptimeHours float64, bpms float64, diversity float64) {
-	hb.Score = (0.4 * uptimeHours) +
-		(0.4 * bpms) +
-		(0.2 * diversity)
+	hb.Score = ((0.3 * uptimeHours) +
+		(0.3 * bpms) +
+		(0.4 * diversity)) * 100
 }

 type HeartbeatInfo []struct {
@@ -433,34 +536,213 @@ type HeartbeatInfo []struct {

 const ProtocolHeartbeat = "/opencloud/heartbeat/1.0"

-func SendHeartbeat(ctx context.Context, proto protocol.ID, name string, h host.Host, ps ProtocolStream, peers map[string]*pp.AddrInfo, interval time.Duration) {
-	peerID, err := oclib.GenerateNodeID()
-	if err == nil {
-		panic("can't heartbeat daemon failed to start")
+// ProtocolBandwidthProbe is a dedicated short-lived stream used exclusively
+// for bandwidth/latency measurement. The handler echoes any bytes it receives.
+// All nodes and indexers register this handler so peers can measure them.
+const ProtocolBandwidthProbe = "/opencloud/probe/1.0"
+
+// HandleBandwidthProbe echoes back everything written on the stream, then closes.
+// It is registered by all participants so the measuring side (the heartbeat receiver)
+// can open a dedicated probe stream and read the round-trip latency + throughput.
+func HandleBandwidthProbe(s network.Stream) {
+	defer s.Close()
+	s.SetDeadline(time.Now().Add(10 * time.Second))
+	io.Copy(s, s) // echo every byte back to the sender
+}
+
+// SendHeartbeat starts a goroutine that sends periodic heartbeats to peers.
+// recordFn, when provided, is called on each tick and its output is embedded in
+// the heartbeat as a fresh signed PeerRecord so the receiving indexer can
+// republish it to the DHT without an extra round-trip.
+// Pass no recordFn (or nil) for indexer→indexer / native heartbeats.
+func SendHeartbeat(ctx context.Context, proto protocol.ID, name string, h host.Host, ps ProtocolStream, peers map[string]*pp.AddrInfo, mu *sync.RWMutex, interval time.Duration, recordFn ...func() json.RawMessage) {
+	logger := oclib.GetLogger()
+	// isIndexerHB is true when this goroutine drives the indexer heartbeat.
+	// isNativeHB is true when it drives the native heartbeat.
+	isIndexerHB := mu == &StreamMuIndexes
+	isNativeHB := mu == &StreamNativeMu
+	var recFn func() json.RawMessage
+	if len(recordFn) > 0 {
+		recFn = recordFn[0]
 	}
 	go func() {
+		logger.Info().Str("proto", string(proto)).Int("peers", len(peers)).Msg("heartbeat started")
 		t := time.NewTicker(interval)
 		defer t.Stop()
+
+		// doTick sends one round of heartbeats to the current peer snapshot.
+		doTick := func() {
+			// Build the heartbeat payload — snapshot current indexer addresses.
+			StreamMuIndexes.RLock()
+			addrs := make([]string, 0, len(StaticIndexers))
+			for addr := range StaticIndexers {
+				addrs = append(addrs, addr)
+			}
+			StreamMuIndexes.RUnlock()
+			hb := Heartbeat{
+				Name:           name,
+				PeerID:         h.ID().String(),
+				Timestamp:      time.Now().UTC().Unix(),
+				IndexersBinded: addrs,
+			}
+			if recFn != nil {
+				hb.Record = recFn()
+			}
+
+			// Snapshot the peer list under a read lock so we don't hold the
+			// write lock during network I/O.
+			if mu != nil {
+				mu.RLock()
+			}
+			snapshot := make([]*pp.AddrInfo, 0, len(peers))
+			for _, ix := range peers {
+				snapshot = append(snapshot, ix)
+			}
+			if mu != nil {
+				mu.RUnlock()
+			}
+
+			for _, ix := range snapshot {
+				wasConnected := h.Network().Connectedness(ix.ID) == network.Connected
+				if err := sendHeartbeat(ctx, h, proto, ix, hb, ps, interval*time.Second); err != nil {
+					// Step 3: heartbeat failed — remove from pool and trigger replenish.
+					logger.Info().Str("peer", ix.ID.String()).Str("proto", string(proto)).Msg("[native] step 3 — heartbeat failed, removing peer from pool")
+
+					// Remove the dead peer and clean up its stream.
+					// mu already covers ps when isIndexerHB (same mutex), so one
+					// lock acquisition is sufficient — no re-entrant double-lock.
+					if mu != nil {
+						mu.Lock()
+					}
+					if ps[proto] != nil {
+						if s, ok := ps[proto][ix.ID]; ok {
+							if s.Stream != nil {
+								s.Stream.Close()
+							}
+							delete(ps[proto], ix.ID)
+						}
+					}
+					lostAddr := ""
+					for addr, ad := range peers {
+						if ad.ID == ix.ID {
+							lostAddr = addr
+							delete(peers, addr)
+							break
+						}
+					}
+					need := conf.GetConfig().MinIndexer - len(peers)
+					remaining := len(peers)
+					if mu != nil {
+						mu.Unlock()
+					}
+					logger.Info().Int("remaining", remaining).Int("min", conf.GetConfig().MinIndexer).Int("need", need).Msg("[native] step 3 — pool state after removal")
+
+					// Step 4: ask the native for the missing indexer count.
+					if isIndexerHB && conf.GetConfig().NativeIndexerAddresses != "" {
+						if need < 1 {
+							need = 1
+						}
+						logger.Info().Int("need", need).Msg("[native] step 3→4 — triggering replenish")
+						go replenishIndexersFromNative(h, need)
+					}
+
+					// Native heartbeat failed — find a replacement native.
+					// Case 1: if the dead native was also serving as an indexer, evict it
+					// from StaticIndexers immediately without waiting for the indexer HB tick.
+					if isNativeHB {
+						logger.Info().Str("addr", lostAddr).Msg("[native] step 3 — native heartbeat failed, triggering native replenish")
+						if lostAddr != "" && conf.GetConfig().NativeIndexerAddresses != "" {
+							StreamMuIndexes.Lock()
+							if _, wasIndexer := StaticIndexers[lostAddr]; wasIndexer {
+								delete(StaticIndexers, lostAddr)
+								if s := StreamIndexers[ProtocolHeartbeat]; s != nil {
+									if stream, ok := s[ix.ID]; ok {
+										if stream.Stream != nil {
+											stream.Stream.Close()
+										}
+										delete(s, ix.ID)
+									}
+								}
+								idxNeed := conf.GetConfig().MinIndexer - len(StaticIndexers)
+								StreamMuIndexes.Unlock()
+								if idxNeed < 1 {
+									idxNeed = 1
+								}
+								logger.Info().Str("addr", lostAddr).Msg("[native] dead native evicted from indexer pool, triggering replenish")
+								go replenishIndexersFromNative(h, idxNeed)
+							} else {
+								StreamMuIndexes.Unlock()
+							}
+						}
+						go replenishNativesFromPeers(h, lostAddr, proto)
+					}
+				} else {
+					// Case 2: native-as-indexer reconnected after a restart.
+					// If the peer was disconnected before this tick and the heartbeat just
+					// succeeded (transparent reconnect), the native may have restarted with
+					// blank state (responsiblePeers empty). Evict it from StaticIndexers and
+					// re-request an assignment so the native re-tracks us properly and
+					// runOffloadLoop can eventually migrate us to real indexers.
+					if !wasConnected && isIndexerHB && conf.GetConfig().NativeIndexerAddresses != "" {
+						StreamNativeMu.RLock()
+						isNativeIndexer := false
+						for _, ad := range StaticNatives {
+							if ad.ID == ix.ID {
+								isNativeIndexer = true
+								break
+							}
+						}
+						StreamNativeMu.RUnlock()
+						if isNativeIndexer {
+							if mu != nil {
+								mu.Lock()
+							}
+							if ps[proto] != nil {
+								if s, ok := ps[proto][ix.ID]; ok {
+									if s.Stream != nil {
+										s.Stream.Close()
+									}
+									delete(ps[proto], ix.ID)
+								}
+							}
+							reconnectedAddr := ""
+							for addr, ad := range peers {
+								if ad.ID == ix.ID {
+									reconnectedAddr = addr
+									delete(peers, addr)
+									break
+								}
+							}
+							idxNeed := conf.GetConfig().MinIndexer - len(peers)
+							if mu != nil {
+								mu.Unlock()
+							}
+							if idxNeed < 1 {
+								idxNeed = 1
+							}
+							logger.Info().Str("addr", reconnectedAddr).Str("peer", ix.ID.String()).Msg(
+								"[native] native-as-indexer reconnected after restart — evicting and re-requesting assignment")
+							go replenishIndexersFromNative(h, idxNeed)
+						}
+					}
+					logger.Debug().Str("peer", ix.ID.String()).Str("proto", string(proto)).Msg("[native] step 2 — heartbeat sent ok")
+				}
+			}
+		}
+
 		for {
 			select {
 			case <-t.C:
-				addrs := []string{}
-				for addr := range StaticIndexers {
-					addrs = append(addrs, addr)
+				doTick()
+			case <-indexerHeartbeatNudge:
+				if isIndexerHB {
+					logger.Info().Msg("[native] step 2 — nudge received, heartbeating new indexers immediately")
+					doTick()
 				}
-				hb := Heartbeat{
-					Name:           name,
-					DID:            peerID,
-					PeerID:         h.ID().String(),
-					Timestamp:      time.Now().UTC().Unix(),
-					IndexersBinded: addrs,
-				}
-				for _, ix := range peers {
-					if err = sendHeartbeat(ctx, h, proto, ix, hb, ps, interval*time.Second); err != nil {
-						StreamMuIndexes.Lock()
-						delete(StreamIndexers[proto], ix.ID)
-						StreamMuIndexes.Unlock()
-					}
+			case <-nativeHeartbeatNudge:
+				if isNativeHB {
+					logger.Info().Msg("[native] native nudge received, heartbeating replacement native immediately")
+					doTick()
 				}
 			case <-ctx.Done():
 				return
@@ -480,58 +762,62 @@ func TempStream(h host.Host, ad pp.AddrInfo, proto protocol.ID, did string, stre
 	if pts[proto] != nil {
 		expiry = pts[proto].TTL
 	}
-	if ctxTTL, err := context.WithTimeout(context.Background(), expiry); err == nil {
-		if h.Network().Connectedness(ad.ID) != network.Connected {
-			if err := h.Connect(ctxTTL, ad); err != nil {
-				return streams, err
-			}
-		}
-		if streams[proto] != nil && streams[proto][ad.ID] != nil {
-			return streams, nil
-		} else if s, err := h.NewStream(ctxTTL, ad.ID, proto); err == nil {
-			mu.Lock()
-			if streams[proto] == nil {
-				streams[proto] = map[pp.ID]*Stream{}
-			}
-			mu.Unlock()
-			time.AfterFunc(expiry, func() {
-				mu.Lock()
-				defer mu.Unlock()
-				delete(streams[proto], ad.ID)
-			})
-			streams[ProtocolPublish][ad.ID] = &Stream{
-				DID:    did,
-				Stream: s,
-				Expiry: time.Now().UTC().Add(expiry),
-			}
-			mu.Unlock()
-			return streams, nil
-		} else {
+	ctxTTL, _ := context.WithTimeout(context.Background(), expiry)
+	if h.Network().Connectedness(ad.ID) != network.Connected {
+		if err := h.Connect(ctxTTL, ad); err != nil {
 			return streams, err
 		}
 	}
-	return streams, errors.New("can't create a context")
+	if streams[proto] != nil && streams[proto][ad.ID] != nil {
+		return streams, nil
+	} else if s, err := h.NewStream(ctxTTL, ad.ID, proto); err == nil {
+		mu.Lock()
+		if streams[proto] == nil {
+			streams[proto] = map[pp.ID]*Stream{}
+		}
+		mu.Unlock()
+		time.AfterFunc(expiry, func() {
+			mu.Lock()
+			delete(streams[proto], ad.ID)
+			mu.Unlock()
+		})
+		mu.Lock()
+		streams[proto][ad.ID] = &Stream{
+			DID:    did,
+			Stream: s,
+			Expiry: time.Now().UTC().Add(expiry),
+		}
+		mu.Unlock()
+		return streams, nil
+	} else {
+		return streams, err
+	}
 }

 func sendHeartbeat(ctx context.Context, h host.Host, proto protocol.ID, p *pp.AddrInfo,
 	hb Heartbeat, ps ProtocolStream, interval time.Duration) error {
-	streams := ps.Get(proto)
-	if len(streams) == 0 {
-		return errors.New("no stream for protocol heartbeat founded")
+	logger := oclib.GetLogger()
+	if ps[proto] == nil {
+		ps[proto] = map[pp.ID]*Stream{}
 	}
+	streams := ps[proto]
 	pss, exists := streams[p.ID]
-	ctxTTL, _ := context.WithTimeout(ctx, 3*interval)
+	ctxTTL, cancel := context.WithTimeout(ctx, 3*interval)
+	defer cancel()
 	// Connect si nécessaire
 	if h.Network().Connectedness(p.ID) != network.Connected {
 		if err := h.Connect(ctxTTL, *p); err != nil {
+			logger.Err(err)
 			return err
 		}
 		exists = false // on devra recréer le stream
 	}
 	// Crée le stream si inexistant ou fermé
 	if !exists || pss.Stream == nil {
+		logger.Info().Msg("New Stream engaged as Heartbeat " + fmt.Sprintf("%v", proto) + " " + p.ID.String())
 		s, err := h.NewStream(ctx, p.ID, proto)
 		if err != nil {
+			logger.Err(err)
 			return err
 		}
 		pss = &Stream{