From 424d523c5e10f8a6fb639c1347ee9b7bca8e4273 Mon Sep 17 00:00:00 2001 From: pb Date: Wed, 9 Jul 2025 16:50:41 +0200 Subject: [PATCH 01/15] added the bson tag for DestPeerId which was causing error when deserializing from mongo results --- models/booking/booking.go | 2 +- models/resources/purchase_resource/purchase_resource.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/booking/booking.go b/models/booking/booking.go index 831f443..bf20a91 100644 --- a/models/booking/booking.go +++ b/models/booking/booking.go @@ -23,7 +23,7 @@ type Booking struct { ExecutionMetrics map[string][]models.MetricsSnapshot `json:"metrics,omitempty" bson:"metrics,omitempty"` ExecutionsID string `json:"executions_id,omitempty" bson:"executions_id,omitempty" validate:"required"` // ExecutionsID is the ID of the executions - DestPeerID string `json:"dest_peer_id,omitempty"` // DestPeerID is the ID of the destination peer + DestPeerID string `json:"dest_peer_id,omitempty" bson:"dest_peer_id,omitempty"` // DestPeerID is the ID of the destination peer WorkflowID string `json:"workflow_id,omitempty" bson:"workflow_id,omitempty"` // WorkflowID is the ID of the workflow ExecutionID string `json:"execution_id,omitempty" bson:"execution_id,omitempty" validate:"required"` State enum.BookingStatus `json:"state,omitempty" bson:"state,omitempty" validate:"required"` // State is the state of the booking diff --git a/models/resources/purchase_resource/purchase_resource.go b/models/resources/purchase_resource/purchase_resource.go index ca528ae..077b076 100644 --- a/models/resources/purchase_resource/purchase_resource.go +++ b/models/resources/purchase_resource/purchase_resource.go @@ -10,7 +10,7 @@ import ( type PurchaseResource struct { utils.AbstractObject - DestPeerID string + DestPeerID string `json:"dest_peer_id" bson:"dest_peer_id"` PricedItem pricing.PricedItemITF `json:"priced_item,omitempty" bson:"priced_item,omitempty" validate:"required"` ExecutionsID string `json:"executions_id,omitempty" bson:"executions_id,omitempty" validate:"required"` // ExecutionsID is the ID of the executions EndDate *time.Time `json:"end_buying_date,omitempty" bson:"end_buying_date,omitempty"` From 83e590d4e190e0c6d7a312713993f340be6ca3ef Mon Sep 17 00:00:00 2001 From: pb Date: Wed, 9 Jul 2025 17:42:37 +0200 Subject: [PATCH 02/15] added error handling --- models/bill/bill.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/models/bill/bill.go b/models/bill/bill.go index aa31062..31e3f1e 100644 --- a/models/bill/bill.go +++ b/models/bill/bill.go @@ -2,6 +2,7 @@ package bill import ( "encoding/json" + "fmt" "sync" "time" @@ -186,7 +187,11 @@ type PeerItemOrder struct { func (d *PeerItemOrder) GetPrice(request *tools.APIRequest) (float64, error) { var priced *resources.PricedResource b, _ := json.Marshal(d.Item) - json.Unmarshal(b, priced) + err := json.Unmarshal(b, priced) + if err != nil { + fmt.Println(err) + return 0, err + } accessor := purchase_resource.NewAccessor(request) search, code, _ := accessor.Search(&dbs.Filters{ And: map[string][]dbs.Filter{ From 98a2359c9d9f1845d7c0414d08ec329fe3d3b6f8 Mon Sep 17 00:00:00 2001 From: pb Date: Thu, 10 Jul 2025 11:47:54 +0200 Subject: [PATCH 03/15] added a tweak to PeerItemOrder GetPrice in order to not crash on nil Purchase --- models/bill/bill.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/models/bill/bill.go b/models/bill/bill.go index 31e3f1e..32bd67e 100644 --- a/models/bill/bill.go +++ b/models/bill/bill.go @@ -185,6 +185,11 @@ type PeerItemOrder struct { } func (d *PeerItemOrder) GetPrice(request *tools.APIRequest) (float64, error) { + /////////// Temporary in order to allow GenerateOrder to complete while billing is still WIP + if d.Purchase == nil { + return 0, nil + } + /////////// var priced *resources.PricedResource b, _ := json.Marshal(d.Item) err := json.Unmarshal(b, priced) From e735f78e58c6f31ed34bbbaeaf2e545cc8969522 Mon Sep 17 00:00:00 2001 From: pb Date: Tue, 15 Jul 2025 14:58:19 +0200 Subject: [PATCH 04/15] added a new method to create kubernetes names with the same naming --- entrypoint.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/entrypoint.go b/entrypoint.go index 7429ffa..7dc00ba 100644 --- a/entrypoint.go +++ b/entrypoint.go @@ -597,3 +597,15 @@ func (l *LibData) ToPurchasedResource() *purchase_resource.PurchaseResource { } return nil } + + +// ============== ADMIRALTY ============== +// Returns a concatenation of the peerId and namespace in order for +// kubernetes ressources to have a unique name, under 63 characters +// and yet identify which peer they are created for +func GetConcatenatedName(peerId string, namespace string) string { + s := strings.Split(namespace, "-")[:2] + n := s[0] + "-" + s[1] + + return peerId + "-" + n +} \ No newline at end of file From 76d83878ebd335237788a657dd10ff0979d9afd4 Mon Sep 17 00:00:00 2001 From: pb Date: Wed, 30 Jul 2025 18:01:23 +0200 Subject: [PATCH 05/15] added a method to workflow which allows to retrieve items by resource type and resource id --- models/workflow/workflow.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/models/workflow/workflow.go b/models/workflow/workflow.go index 71f0a1f..b9f456b 100644 --- a/models/workflow/workflow.go +++ b/models/workflow/workflow.go @@ -227,6 +227,27 @@ func (wf *Workflow) Planify(start time.Time, end *time.Time, request *tools.APIR return longest, priceds, wf, nil } +func (w *Workflow) GetItemsByResources() (res map[tools.DataType]map[string][]string) { + dtMethodMap := map[tools.DataType]func() []graph.GraphItem{ + tools.STORAGE_RESOURCE: func() []graph.GraphItem { return w.GetGraphItems(w.Graph.IsStorage) }, + tools.DATA_RESOURCE: func() []graph.GraphItem { return w.GetGraphItems(w.Graph.IsData) }, + tools.COMPUTE_RESOURCE: func() []graph.GraphItem { return w.GetGraphItems(w.Graph.IsCompute) }, + tools.PROCESSING_RESOURCE: func() []graph.GraphItem { return w.GetGraphItems(w.Graph.IsProcessing) }, + tools.WORKFLOW_RESOURCE: func() []graph.GraphItem { return w.GetGraphItems(w.Graph.IsWorkflow) }, + } + + for dt, meth := range dtMethodMap { + items := meth() + for _, i := range items { + _, r := i.GetResource() + rId := r.GetID() + res[dt][rId] = append(res[dt][rId],i.ID) + } + } + + return +} + func plan[T resources.ResourceInterface]( dt tools.DataType, wf *Workflow, priceds map[tools.DataType]map[string]pricing.PricedItemITF, request *tools.APIRequest, f func(graph.GraphItem) bool, start func(resources.ResourceInterface, pricing.PricedItemITF) (time.Time, float64), end func(time.Time, float64) *time.Time) ([]T, map[tools.DataType]map[string]pricing.PricedItemITF, error) { From a093369dc5a2cf650fbe7371ed48224a8c9b4da9 Mon Sep 17 00:00:00 2001 From: pb Date: Wed, 30 Jul 2025 18:15:55 +0200 Subject: [PATCH 06/15] corrction of the non initialized map --- models/workflow/workflow.go | 1 + 1 file changed, 1 insertion(+) diff --git a/models/workflow/workflow.go b/models/workflow/workflow.go index b9f456b..68ff7fc 100644 --- a/models/workflow/workflow.go +++ b/models/workflow/workflow.go @@ -237,6 +237,7 @@ func (w *Workflow) GetItemsByResources() (res map[tools.DataType]map[string][]st } for dt, meth := range dtMethodMap { + res[dt] = make(map[string][]string) items := meth() for _, i := range items { _, r := i.GetResource() From be2a1cc11474074ae3fe87f7b970a287702b1378 Mon Sep 17 00:00:00 2001 From: pb Date: Wed, 30 Jul 2025 18:21:09 +0200 Subject: [PATCH 07/15] corrction of the non initialized map --- models/workflow/workflow.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/models/workflow/workflow.go b/models/workflow/workflow.go index 68ff7fc..49f99ad 100644 --- a/models/workflow/workflow.go +++ b/models/workflow/workflow.go @@ -227,7 +227,8 @@ func (wf *Workflow) Planify(start time.Time, end *time.Time, request *tools.APIR return longest, priceds, wf, nil } -func (w *Workflow) GetItemsByResources() (res map[tools.DataType]map[string][]string) { +func (w *Workflow) GetItemsByResources() (map[tools.DataType]map[string][]string) { + res := make(map[tools.DataType]map[string][]string) dtMethodMap := map[tools.DataType]func() []graph.GraphItem{ tools.STORAGE_RESOURCE: func() []graph.GraphItem { return w.GetGraphItems(w.Graph.IsStorage) }, tools.DATA_RESOURCE: func() []graph.GraphItem { return w.GetGraphItems(w.Graph.IsData) }, @@ -246,7 +247,7 @@ func (w *Workflow) GetItemsByResources() (res map[tools.DataType]map[string][]st } } - return + return res } func plan[T resources.ResourceInterface]( From 3ddbf1a9674a6ab359308ae1f2e27efe86ad5c50 Mon Sep 17 00:00:00 2001 From: pb Date: Wed, 30 Jul 2025 18:28:19 +0200 Subject: [PATCH 08/15] added comments --- models/workflow/workflow.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/models/workflow/workflow.go b/models/workflow/workflow.go index 49f99ad..880d720 100644 --- a/models/workflow/workflow.go +++ b/models/workflow/workflow.go @@ -227,6 +227,10 @@ func (wf *Workflow) Planify(start time.Time, end *time.Time, request *tools.APIR return longest, priceds, wf, nil } +// Returns a map of DataType (processing,computing,data,storage,worfklow) where each resource (identified by its UUID) +// is mapped to the list of its items (different appearance) in the graph +// ex: if the same Minio storage is represented by several nodes in the graph, in [tools.STORAGE_RESSOURCE] its UUID will be mapped to +// the list of GraphItem ID that correspond to the ID of each node func (w *Workflow) GetItemsByResources() (map[tools.DataType]map[string][]string) { res := make(map[tools.DataType]map[string][]string) dtMethodMap := map[tools.DataType]func() []graph.GraphItem{ From cc3091d401ea24514ffb129d14c72e6297eca733 Mon Sep 17 00:00:00 2001 From: pb Date: Thu, 31 Jul 2025 15:53:05 +0200 Subject: [PATCH 09/15] added the function to load one ressource for each ressource type --- entrypoint.go | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/entrypoint.go b/entrypoint.go index 7dc00ba..25db0cf 100644 --- a/entrypoint.go +++ b/entrypoint.go @@ -608,4 +608,54 @@ func GetConcatenatedName(peerId string, namespace string) string { n := s[0] + "-" + s[1] return peerId + "-" + n +} + +// ------------- Loading resources ---------- + +func LoadOneStorage(storageId string, user string, peerID string, groups []string) (*resources.StorageResource, error) { + + res := NewRequest(LibDataEnum(STORAGE_RESOURCE), user, peerID, groups,nil).LoadOne(storageId) + if res.Code != 200 { + l := GetLogger() + l.Error().Msg("Error while loading storage ressource " + storageId) + return nil,fmt.Errorf(res.Err) + } + + return res.ToStorageResource(), nil +} + +func LoadOneComputing(computingId string, user string, peerID string, groups []string) (*resources.ComputeResource, error) { + + res := NewRequest(LibDataEnum(COMPUTE_RESOURCE), user, peerID, groups,nil).LoadOne(computingId) + if res.Code != 200 { + l := GetLogger() + l.Error().Msg("Error while loading computing ressource " + computingId) + return nil,fmt.Errorf(res.Err) + } + + return res.ToComputeResource(), nil +} + +func LoadOneProcessing(processingId string, user string, peerID string, groups []string) (*resources.ProcessingResource, error) { + + res := NewRequest(LibDataEnum(PROCESSING_RESOURCE), user, peerID, groups,nil).LoadOne(processingId) + if res.Code != 200 { + l := GetLogger() + l.Error().Msg("Error while loading processing ressource " + processingId) + return nil,fmt.Errorf(res.Err) + } + + return res.ToProcessingResource(), nil +} + +func LoadOneData(dataId string, user string, peerID string, groups []string) (*resources.DataResource, error) { + + res := NewRequest(LibDataEnum(DATA_RESOURCE), user, peerID, groups,nil).LoadOne(dataId) + if res.Code != 200 { + l := GetLogger() + l.Error().Msg("Error while loading data ressource " + dataId) + return nil,fmt.Errorf(res.Err) + } + return res.ToDataResource(), nil + } \ No newline at end of file From 76e9b2562e9bffdd4d67762b1b82587c21b1402b Mon Sep 17 00:00:00 2001 From: pb Date: Tue, 5 Aug 2025 11:56:27 +0200 Subject: [PATCH 10/15] added the enum to reach the /minio/secet route --- tools/enums.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/enums.go b/tools/enums.go index 5bc0620..58cfa54 100644 --- a/tools/enums.go +++ b/tools/enums.go @@ -30,6 +30,7 @@ const ( LIVE_STORAGE BILL MINIO_SVCACC + MINIO_SVCACC_SECRET ) var NOAPI = "" @@ -75,6 +76,7 @@ var DefaultAPI = [...]string{ DATACENTERAPI, NOAPI, MINIO, + MINIO, } // Bind the standard data name to the data type @@ -105,6 +107,7 @@ var Str = [...]string{ "live_storage", "bill", "service_account", + "secret", } func FromInt(i int) string { From cc939451fd81c95ec4d3c56326fa33d0197de6d4 Mon Sep 17 00:00:00 2001 From: pb Date: Tue, 5 Aug 2025 13:25:47 +0200 Subject: [PATCH 11/15] added the UUID of the peer when an error appears --- models/workflow_execution/workflow_scheduler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/workflow_execution/workflow_scheduler.go b/models/workflow_execution/workflow_scheduler.go index 88e289f..83ef02e 100755 --- a/models/workflow_execution/workflow_scheduler.go +++ b/models/workflow_execution/workflow_scheduler.go @@ -142,7 +142,7 @@ func getBooking(b *booking.Booking, request *tools.APIRequest, errCh chan error, _, err = (&peer.Peer{}).LaunchPeerExecution(b.DestPeerID, b.ResourceID, tools.BOOKING, tools.GET, nil, &c) if err != nil { - errCh <- err + errCh <- fmt.Errorf("error on " + b.DestPeerID ,err) return } From 40a61387b9f165782c75d51bd0c7e6c6d6ad93f1 Mon Sep 17 00:00:00 2001 From: pb Date: Tue, 5 Aug 2025 13:39:21 +0200 Subject: [PATCH 12/15] added the UUID of the peer when an error appears --- models/workflow_execution/workflow_scheduler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/workflow_execution/workflow_scheduler.go b/models/workflow_execution/workflow_scheduler.go index 83ef02e..7dc766b 100755 --- a/models/workflow_execution/workflow_scheduler.go +++ b/models/workflow_execution/workflow_scheduler.go @@ -142,7 +142,7 @@ func getBooking(b *booking.Booking, request *tools.APIRequest, errCh chan error, _, err = (&peer.Peer{}).LaunchPeerExecution(b.DestPeerID, b.ResourceID, tools.BOOKING, tools.GET, nil, &c) if err != nil { - errCh <- fmt.Errorf("error on " + b.DestPeerID ,err) + errCh <- fmt.Errorf("error on " + b.DestPeerID + err.Error()) return } From e7a71188a3b52a687007662230bf94da0c3c4715 Mon Sep 17 00:00:00 2001 From: pb Date: Fri, 8 Aug 2025 16:05:36 +0200 Subject: [PATCH 13/15] changed the value used as the key in the related map, using the 'node id' was erasing some relations when a a processing is linked to two storages --- models/workflow/workflow.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/workflow/workflow.go b/models/workflow/workflow.go index 880d720..5f23b21 100644 --- a/models/workflow/workflow.go +++ b/models/workflow/workflow.go @@ -110,13 +110,13 @@ func (w *Workflow) GetByRelatedProcessing(processingID string, g func(item graph _, node = item.GetResource() // we are looking for the storage as destination } if processingID == nodeID && node != nil { // if the storage is linked to the processing - if _, ok := related[processingID]; !ok { + if _, ok := related[node.GetID()]; !ok { related[processingID] = Related{} } rel := related[node.GetID()] rel.Node = node rel.Links = append(rel.Links, link) - related[processingID] = rel + related[node.GetID()] = rel } } return related From f4b0cf5683de5ae34355febbe1c7c775141f3dd8 Mon Sep 17 00:00:00 2001 From: pb Date: Fri, 8 Aug 2025 16:15:53 +0200 Subject: [PATCH 14/15] changed the value used as the key in the related map, using the 'node id' was erasing some relations when a a processing is linked to two storages --- models/workflow/workflow.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/models/workflow/workflow.go b/models/workflow/workflow.go index 5f23b21..61a70ad 100644 --- a/models/workflow/workflow.go +++ b/models/workflow/workflow.go @@ -110,13 +110,11 @@ func (w *Workflow) GetByRelatedProcessing(processingID string, g func(item graph _, node = item.GetResource() // we are looking for the storage as destination } if processingID == nodeID && node != nil { // if the storage is linked to the processing - if _, ok := related[node.GetID()]; !ok { - related[processingID] = Related{} - } - rel := related[node.GetID()] + relID := node.GetID() + rel := Related{} rel.Node = node rel.Links = append(rel.Links, link) - related[node.GetID()] = rel + related[relID] = rel } } return related From 188b758f7a1f5885e0a4b7a34ba2eb4a4228140c Mon Sep 17 00:00:00 2001 From: root Date: Sat, 1 Nov 2025 16:39:25 +0100 Subject: [PATCH 15/15] Ajouter .gitattributes --- .gitattributes | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..7ddc7f2 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Force Go as the main language +*.go linguist-detectable=true +* linguist-language=Go