core/oc-discovery
6
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Peer Discovery -> DHT // no more pubsub state

Browse Source
This commit is contained in:
mr
2026-02-18 13:29:50 +01:00
parent 6a5ffb9a92
commit 0250c3b339
9 changed files with 318 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

128
daemons/node/indexer/handler.go
View File

@@ -6,7 +6,6 @@ import (
"encoding/json"
"errors"
"fmt"
"oc-discovery/conf"
"oc-discovery/daemons/node/common"
"time"
@@ -30,9 +29,6 @@ type PeerRecord struct {
WalletAddress string `json:"wallet_address"`
Signature []byte `json:"signature"`
ExpiryDate time.Time `json:"expiry_date"`
TTL int `json:"ttl"` // max of hop diffusion
NoPub bool `json:"no_pub"`
}
func (p *PeerRecord) Sign() error {
@@ -118,7 +114,8 @@ func (pr *PeerRecord) ExtractPeer(ourkey string, key string, pubKey crypto.PubKe
}
type GetValue struct {
Key string `json:"key"`
Key string `json:"key"`
PeerID peer.ID `json:"peer_id"`
}
type GetResponse struct {
@@ -126,8 +123,11 @@ type GetResponse struct {
Records map[string]PeerRecord `json:"records,omitempty"`
}
func (ix *IndexerService) genKey(did string) string {
return "/node/" + did
}
func (ix *IndexerService) initNodeHandler() {
fmt.Println("Node activity")
ix.Host.SetStreamHandler(common.ProtocolHeartbeat, ix.HandleNodeHeartbeat)
ix.Host.SetStreamHandler(common.ProtocolPublish, ix.handleNodePublish)
ix.Host.SetStreamHandler(common.ProtocolGet, ix.handleNodeGet)
@@ -139,6 +139,7 @@ func (ix *IndexerService) handleNodePublish(s network.Stream) {
for {
var rec PeerRecord
if err := json.NewDecoder(s).Decode(&rec); err != nil {
logger.Err(err)
continue
}
rec2 := PeerRecord{
@@ -152,7 +153,7 @@ func (ix *IndexerService) handleNodePublish(s network.Stream) {
continue
}
if rec.PeerID == "" || rec.ExpiryDate.Before(time.Now().UTC()) { // already expired
logger.Warn().Msg(rec.PeerID + " is expired.")
logger.Err(errors.New(rec.PeerID + " is expired."))
continue
}
pid, err := peer.Decode(rec.PeerID)
@@ -162,13 +163,12 @@ func (ix *IndexerService) handleNodePublish(s network.Stream) {
ix.StreamMU.Lock()
if ix.StreamRecords[common.ProtocolPublish] == nil {
ix.StreamRecords[common.ProtocolPublish] = map[peer.ID]*common.StreamRecord[PeerRecord]{}
if ix.StreamRecords[common.ProtocolHeartbeat] == nil {
ix.StreamRecords[common.ProtocolHeartbeat] = map[peer.ID]*common.StreamRecord[PeerRecord]{}
}
streams := ix.StreamRecords[common.ProtocolPublish]
streams := ix.StreamRecords[common.ProtocolHeartbeat]
if srec, ok := streams[pid]; ok {
fmt.Println("UPDATE PUBLISH", pid)
srec.DID = rec.DID
srec.Record = rec
srec.HeartbeatStream.UptimeTracker.LastSeen = time.Now().UTC()
@@ -179,36 +179,22 @@ func (ix *IndexerService) handleNodePublish(s network.Stream) {
}
ix.StreamMU.Unlock()
if ix.LongLivedPubSubs[common.TopicPubSubNodeActivity] != nil && !rec.NoPub {
if b, err := json.Marshal(common.TopicNodeActivityPub{
Disposer: "/ip4/" + conf.GetConfig().Hostname + "/tcp/" + fmt.Sprintf("%v", conf.GetConfig().NodeEndpointPort) + "/p2p/" + ix.Host.ID().String(),
DID: rec.DID,
Name: rec.Name,
PeerID: pid.String(),
NodeActivity: pp.ONLINE.EnumIndex(),
}); err == nil {
ix.LongLivedPubSubs[common.TopicPubSubNodeActivity].Publish(context.Background(), b)
}
key := ix.genKey(rec.DID)
data, err := json.Marshal(rec)
if err != nil {
logger.Err(err)
continue
}
if rec.TTL > 0 {
rec.NoPub = true
for _, ad := range common.StaticIndexers {
if ad.ID == s.Conn().RemotePeer() {
continue
}
if common.StreamIndexers[common.ProtocolPublish][ad.ID] == nil {
continue
}
stream := common.StreamIndexers[common.ProtocolPublish][ad.ID]
rec.TTL -= 1
if err := json.NewEncoder(stream.Stream).Encode(&rec); err != nil { // then publish on stream
continue
}
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
if err := ix.DHT.PutValue(ctx, key, data); err != nil {
logger.Err(err)
cancel()
continue
}
cancel()
break // response... so quit
}
}
@@ -223,64 +209,40 @@ func (ix *IndexerService) handleNodeGet(s network.Stream) {
}
ix.StreamMU.Lock()
if ix.StreamRecords[common.ProtocolGet] == nil {
ix.StreamRecords[common.ProtocolGet] = map[peer.ID]*common.StreamRecord[PeerRecord]{}
if ix.StreamRecords[common.ProtocolHeartbeat] == nil {
ix.StreamRecords[common.ProtocolHeartbeat] = map[peer.ID]*common.StreamRecord[PeerRecord]{}
}
resp := GetResponse{
Found: false,
Records: map[string]PeerRecord{},
}
streams := ix.StreamRecords[common.ProtocolPublish]
streams := ix.StreamRecords[common.ProtocolHeartbeat]
key := ix.genKey(req.Key)
// simple lookup by PeerID (or DID)
for _, rec := range streams {
if rec.Record.DID == req.Key || rec.Record.PeerID == req.Key || rec.Record.Name == req.Key { // OK
resp.Found = true
resp.Records[rec.Record.PeerID] = rec.Record
if rec.Record.DID == req.Key || rec.Record.PeerID == req.Key { // there unique... no need to proceed more...
_ = json.NewEncoder(s).Encode(resp)
ix.StreamMU.Unlock()
return
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
recBytes, err := ix.DHT.SearchValue(ctx, key)
if err != nil {
logger.Err(err).Msg("Failed to fetch PeerRecord from DHT")
cancel()
}
cancel()
for c := range recBytes {
var rec PeerRecord
if err := json.Unmarshal(c, &rec); err != nil || rec.PeerID != req.PeerID.String() {
continue
}
}
// if not found ask to my neighboor indexers
for pid, dsp := range ix.DisposedPeers {
if _, ok := resp.Records[dsp.PeerID]; !ok && (dsp.Name == req.Key || dsp.DID == req.Key || dsp.PeerID == req.Key) {
ctxTTL, err := context.WithTimeout(context.Background(), 120*time.Second)
if err != nil {
continue
}
if ix.Host.Network().Connectedness(pid) != network.Connected {
if ad, err := peer.AddrInfoFromString(dsp.Disposer); err == nil {
_ = ix.Host.Connect(ctxTTL, *ad)
str, err := ix.Host.NewStream(ctxTTL, pid, common.ProtocolGet)
if err != nil {
continue
}
for {
if ctxTTL.Err() == context.DeadlineExceeded {
break
}
var subResp GetResponse
if err := json.NewDecoder(str).Decode(&resp); err != nil {
continue
}
if subResp.Found {
for k, v := range subResp.Records {
if _, ok := resp.Records[k]; !ok {
resp.Records[k] = v
}
}
break
}
}
}
}
resp.Found = true
resp.Records[rec.PeerID] = rec
if srec, ok := streams[req.PeerID]; ok {
srec.DID = rec.DID
srec.Record = rec
srec.HeartbeatStream.UptimeTracker.LastSeen = time.Now().UTC()
}
}
// Not found
_ = json.NewEncoder(s).Encode(resp)
ix.StreamMU.Unlock()
break // response... so quit
}
}

34
daemons/node/indexer/service.go
View File

@@ -6,8 +6,9 @@ import (
"sync"
oclib "cloud.o-forge.io/core/oc-lib"
pp "cloud.o-forge.io/core/oc-lib/models/peer"
dht "github.com/libp2p/go-libp2p-kad-dht"
pubsub "github.com/libp2p/go-libp2p-pubsub"
record "github.com/libp2p/go-libp2p-record"
"github.com/libp2p/go-libp2p/core/host"
"github.com/libp2p/go-libp2p/core/peer"
)
@@ -16,9 +17,13 @@ import (
type IndexerService struct {
*common.LongLivedStreamRecordedService[PeerRecord]
PS *pubsub.PubSub
DHT *dht.IpfsDHT
isStrictIndexer bool
mu sync.RWMutex
DisposedPeers map[peer.ID]*common.TopicNodeActivityPub
SeenQueries map[string]bool
SeenMU sync.Mutex
}
// if a pubsub is given... indexer is also an active oc-node. If not... your a strict indexer
@@ -46,28 +51,25 @@ func NewIndexerService(h host.Host, ps *pubsub.PubSub, maxNode int) *IndexerServ
logger.Info().Msg("subscribe to decentralized search flow as strict indexer...")
ix.SubscribeToSearch(ix.PS, nil)
}
f := func(ctx context.Context, evt common.TopicNodeActivityPub, _ string) {
ix.mu.Lock()
if pid, err := peer.Decode(evt.PeerID); err == nil {
if evt.NodeActivity == pp.OFFLINE.EnumIndex() {
delete(ix.DisposedPeers, pid)
}
if evt.NodeActivity == pp.ONLINE.EnumIndex() {
ix.DisposedPeers[pid] = &evt
}
}
ix.mu.Unlock()
if ix.DHT, err = dht.New(
context.Background(),
ix.Host,
dht.Mode(dht.ModeServer),
dht.Validator(record.NamespacedValidator{
"node": PeerRecordValidator{},
}),
); err != nil {
return nil
}
ix.SubscribeToNodeActivity(ix.PS, &f) // now we subscribe to a long run topic named node-activity, to relay message.
ix.initNodeHandler() // then listen up on every protocol expected
ix.initNodeHandler() // then listen up on every protocol expected
return ix
}
func (ix *IndexerService) Close() {
ix.DHT.Close()
ix.PS.UnregisterTopicValidator(common.TopicPubSubSearch)
for _, s := range ix.StreamRecords {
for _, ss := range s {
ss.Stream.Close()
ss.HeartbeatStream.Stream.Close()
}
}

61
daemons/node/indexer/validator.go Normal file
View File

@@ -0,0 +1,61 @@
package indexer
import (
"encoding/json"
"errors"
"time"
)
type PeerRecordValidator struct{}
func (v PeerRecordValidator) Validate(key string, value []byte) error {
var rec PeerRecord
if err := json.Unmarshal(value, &rec); err != nil {
return errors.New("invalid json")
}
// PeerID must exist
if rec.PeerID == "" {
return errors.New("missing peerID")
}
// Expiry check
if rec.ExpiryDate.Before(time.Now().UTC()) {
return errors.New("record expired")
}
// Signature verification
rec2 := PeerRecord{
Name: rec.Name,
DID: rec.DID,
PubKey: rec.PubKey,
PeerID: rec.PeerID,
}
if _, err := rec2.Verify(); err != nil {
return errors.New("invalid signature")
}
return nil
}
func (v PeerRecordValidator) Select(key string, values [][]byte) (int, error) {
var newest time.Time
index := 0
for i, val := range values {
var rec PeerRecord
if err := json.Unmarshal(val, &rec); err != nil {
continue
}
if rec.ExpiryDate.After(newest) {
newest = rec.ExpiryDate
index = i
}
}
return index, nil
}