17 Commits

Author SHA1 Message Date
pb
d91afc2acd New class link to handle links between components and possible errors 2024-04-16 16:12:54 +02:00
pb
99278ff7ef minor indentation change 2024-04-15 11:43:07 +02:00
pb
64ae7eeb4c added links models to workflow 2024-04-15 11:42:17 +02:00
pb
ff9021b1ff differentiate links and cells post unmarshalling 2024-04-12 13:25:01 +02:00
pb
a07c7f2898 minor changes 2024-04-11 17:06:22 +02:00
pb
cba7b39216 minor changes 2024-04-11 17:06:22 +02:00
pb
590a5070e9 improved UML documentation 2024-04-11 17:06:22 +02:00
pb
57438ed215 removed wrong logic from previous change 2024-04-11 17:06:22 +02:00
pb
b51c0f1b74 update the object stored for one workflow with user input 2024-04-11 17:06:22 +02:00
pb
49d3ba9763 Retrieve component from mxCell inside object tag 2024-04-11 17:06:22 +02:00
pb
83c3c3d3f2 fixed go.mod 2024-04-11 17:06:22 +02:00
pb
1d2638f855 decomment some code to allow build 2024-04-11 17:06:22 +02:00
pb
baada840bf decomment some code to allow build 2024-04-11 17:06:22 +02:00
pb
c2c1c4827f remove thumbs.db:encryptable 2024-04-11 17:06:22 +02:00
pb
8c5e75855e new model for links 2024-04-11 16:34:43 +02:00
pb
9cd8af282f added some doc 2024-04-09 15:40:03 +02:00
pb
dbde933b74 fixed a typo 2024-04-09 15:39:32 +02:00
15 changed files with 983 additions and 156 deletions

View File

@@ -13,7 +13,7 @@ RUN bee generate routers
# Generating the swagger
RUN timeout 20 bee run -gendoc=true -downdoc=true -runmode=dev || :
RUN sed -i 's/http:\/\/127.0.0.1:8080\/swagger\/swagger.json/swagger.json/g' swagger/index.html && \
RUN sed -i 's/http:\/\/127.0.0.1:8080\/swagger\/swagger.json/swagger.json/g' swagger/index.html && \
sed -i 's/https:\/\/petstore.swagger.io\/v2\/swagger.json/swagger.json/g' swagger/index.html
RUN ls -l routers

View File

@@ -63,4 +63,37 @@ This script should be updated to be ran from anywhere.
# More documentation
[Visit the docs/ directory](/docs/)
[Visit the docs/ directory](/docs/)
## UML
We are currently using [goplantuml](https://github.com/jfeliu007/goplantuml/) to generate the oject diagrams directly from the go files. This tools creates `.plums` files directly from the directory where the go files are located. These UML files can then be visualized using plantuml and the VS Code extensions plantuml.
**Setting up the plantuml environment** :
```
# Downloading the go tool goplantuml
go get github.com/jfeliu007/goplantuml/parser
go install github.com/jfeliu007/goplantuml/cmd/goplantuml@latest
# Install the plantuml environment
sudo apt install default-jre
sudo apt install plantuml
```
**Rich PlantUML extension Link**: https://marketplace.visualstudio.com/items?itemName=jebbs.plantuml or search it in the marketplace panel of VS Code
**Generate and view an UML diagram**
```
goplantuml path/to/code/directory > path/to/dest/file.puml
```
- open the .puml file generated
- alt + d to have the extension preview the plantUML code
- right click inside the .puml and select 'Export Current Diagram' to create an image file
- the output will be generated in a `out/` directory in the current path
### Note
Only `.puml` files ought to be commited to the git repository, because their nature allows to track changes, while image file do not fit the purpose of versioning.

View File

@@ -0,0 +1,47 @@
@startuml class_links
class LinkGraph {
+ []Link links
+ void AddLinkToGraph(Link link)
+ boolean,str HasPrevious(Link)
+ boolean,str HasNext(Link)
}
class Link {
+ str source
+ str destination
+ boolean DCLink
+ *Link NewLink(interface{} src, interface{} dst)
+ void AddLinkToDataCenter()
}
note left of LinkGraph::HasPrevious
checks if the component whose ID is in src is the dst of
any other Link of the list on Link
end note
note top of Link
Links need to be redefined in the sense that they are currently used
both :
- to connect a component to a DC
- to represent the interractions between components
end note
note right of Link::DCLink
set at construction of the object and used to order the links
end note
note left of Link::NewLink
Must test if the parameters check the type constraints
and raise errors for the GUI if necessary
end note
LinkGraph*--"links"Link
@enduml

View File

@@ -0,0 +1,77 @@
@startuml
namespace controllers {
class ComputingController << (S,Aquamarine) >> {
+ GetOneComputing(ID string)
+ PostComputing(body models.ComputingNEWModel)
+ GetMultipleComputing(IDs []string)
}
class DataController << (S,Aquamarine) >> {
+ GetOneData(ID string)
+ GetMultipleData(IDs []string)
+ PostData(body models.DataNEWModel)
}
class DatacenterController << (S,Aquamarine) >> {
+ GetMultipleDatacenter(IDs []string)
+ GetOneDatacenter(ID string)
+ PostDatacenter(body models.DatacenterNEWModel)
}
class ScheduleController << (S,Aquamarine) >> {
+ CreateSchedule(dcName string, workflowName string, cron string, duration uint, startDate time.Time, stopDate time.Time, requirements models.ExecutionRequirementsModel)
+ CheckSchedule(cron string, duration uint, startDate time.Time, stopDate time.Time, requirements models.ExecutionRequirementsModel)
+ GetSchedules(startDate time.Time, stopDate time.Time)
+ GetNextSchedules(baseDate time.Time)
+ GetPreviousSchedules(baseDate time.Time)
}
class SearchController << (S,Aquamarine) >> {
+ FindByWord(word string)
}
class StorageController << (S,Aquamarine) >> {
+ GetOneStorage(ID string)
+ GetMultipleStorage(IDs []string)
+ PostStorage(body models.StorageNEWModel)
}
class UserController << (S,Aquamarine) >> {
+ Login()
+ Logout()
}
class WorkflowController << (S,Aquamarine) >> {
+ CreateWorkflow(workflowName string)
+ ListWorkflows()
+ GetWorkflow(workflowName string)
+ AddElementWorkflow(workflowName string, rID string)
+ MxGraphParser(workflowName string, xmlData string)
+ MxGraphParserConsume(workflowName string)
+ LinkElementsWorkflow(workflowName string, rObjIDsource string, rObjIDtarger string, isInput bool)
+ GetWorkflowSchedule(workflowName string)
+ SetWorkflowSchedule(workflowName string, cronString string, events string, isService bool, startDate time.Time, stopDate time.Time, duration uint)
+ CheckWorkflowSchedule(workflowName string)
+ BookWorkflowSchedule(workflowName string)
}
class WorkspaceController << (S,Aquamarine) >> {
+ AddModel(id string, rtype string)
+ ListWorkspace()
+ ListWorkspaceModel()
+ DeleteElement(id string, rtype string)
}
}
"web.Controller" *-- "controllers.ComputingController"
"web.Controller" *-- "controllers.DataController"
"web.Controller" *-- "controllers.DatacenterController"
"web.Controller" *-- "controllers.ScheduleController"
"web.Controller" *-- "controllers.SearchController"
"web.Controller" *-- "controllers.StorageController"
"web.Controller" *-- "controllers.UserController"
"web.Controller" *-- "controllers.WorkflowController"
"web.Controller" *-- "controllers.WorkspaceController"
@enduml

View File

@@ -0,0 +1,347 @@
@startuml
namespace models {
class ComputingModel << (S,Aquamarine) >> {
+ ID string
- getRtype() rtype.Rtype
- getName() string
+ AddUserInput(inputs <font color=blue>map</font>[string]<font color=blue>interface</font>{})
}
class ComputingNEWModel << (S,Aquamarine) >> {
+ Name string
+ Description string
+ ShortDescription string
+ Logo string
+ Type string
+ Owner string
+ License string
+ Price uint
+ ExecutionRequirements ExecutionRequirementsModel
+ Dinputs []string
+ Doutputs []string
+ Image string
+ Command string
+ Arguments []string
+ Environment []string
+ Ports []string
}
class ComputingObject << (S,Aquamarine) >> {
+ ReferenceID primitive.ObjectID
+ Inputs []string
+ Outputs []string
+ DataCenterID string
- getHost() *string
- setReference(rID primitive.ObjectID)
- getReference() primitive.ObjectID
- getRtype() rtype.Rtype
- getModel() (ResourceModel, error)
- getName() *string
- isLinked(rObjID string) LinkingState
- addLink(direction LinkingState, rID string)
}
class DCstatus << (S,Aquamarine) >> {
+ DCname string
+ DCobjID string
+ IsReachable bool
+ IsAvailable bool
+ Booked *ScheduleInfo
+ ErrorMessage string
}
class DataIO << (S,Aquamarine) >> {
+ Counter uint
}
class DataModel << (S,Aquamarine) >> {
+ ID string
- getRtype() rtype.Rtype
- getName() string
}
class DataNEWModel << (S,Aquamarine) >> {
+ Name string
+ Description string
+ ShortDescription string
+ Logo string
+ Dtype string
+ Type string
+ Example string
+ Protocol []string
+ Location string
}
class DataObject << (S,Aquamarine) >> {
+ ReferenceID primitive.ObjectID
- getHost() *string
- getModel() (ResourceModel, error)
- setReference(rID primitive.ObjectID)
- getReference() primitive.ObjectID
- getRtype() rtype.Rtype
- getName() *string
- isLinked(rID string) LinkingState
- addLink(direction LinkingState, rObjID string)
}
class DatacenterCpuModel << (S,Aquamarine) >> {
+ Cores uint
+ Architecture string
+ Shared bool
+ MinimumMemory uint
+ Platform string
}
class DatacenterGpuModel << (S,Aquamarine) >> {
+ CudaCores uint
+ Model string
+ Memory uint
+ TensorCores uint
}
class DatacenterMemoryModel << (S,Aquamarine) >> {
+ Size uint
+ Ecc bool
}
class DatacenterModel << (S,Aquamarine) >> {
+ ID string
- getRtype() rtype.Rtype
- getName() string
+ GetTotalCPUs() uint
+ GetTotalGPUs() uint
+ GetTotalRAM() uint
}
class DatacenterNEWModel << (S,Aquamarine) >> {
+ Name string
+ Type string
+ Acronym string
+ Hosts []string
+ Description string
+ ShortDescription string
+ Logo string
+ CPU DatacenterCpuModel
+ RAM DatacenterMemoryModel
+ GPU []DatacenterGpuModel
+ Owner string
+ BookingPrice int
}
class DatacenterObject << (S,Aquamarine) >> {
+ ReferenceID primitive.ObjectID
- setReference(rID primitive.ObjectID)
- getModel() (ResourceModel, error)
- getReference() primitive.ObjectID
- getHost() *string
- getRtype() rtype.Rtype
- getName() *string
- isLinked(rID string) LinkingState
- addLink(direction LinkingState, rObjID string)
}
class ExecutionRequirementsModel << (S,Aquamarine) >> {
+ CPUs uint
+ GPUs uint
+ RAM uint
+ Parallel bool
+ ScalingModel uint
+ DiskIO string
}
class MxCell << (S,Aquamarine) >> {
+ XMLName xml.Name
+ ID string
+ Parent *string
+ RID *string
+ Source *string
+ Target *string
}
class MxGraphModel << (S,Aquamarine) >> {
+ XMLName xml.Name
+ Root <font color=blue>struct</font>{xml.Name, []MxCell, *[]Object}
}
class Object << (S,Aquamarine) >> {
+ XMLName xml.Name
+ ID string
+ Command *string
+ Args *string
+ Env *string
+ MxCell MxCell
}
class RepositoryModel << (S,Aquamarine) >> {
+ Credentials string
+ Url string
}
interface ResourceModel {
- getRtype() rtype.Rtype
- getName() string
}
interface ResourceObject {
- getHost() *string
- getName() *string
- getModel() (ResourceModel, error)
- getRtype() rtype.Rtype
- setReference(rObjID primitive.ObjectID)
- getReference() primitive.ObjectID
- isLinked(rObjID string) LinkingState
- addLink(direction LinkingState, rObjID string)
}
class ScheduleDB << (S,Aquamarine) >> {
+ StartDate time.Time
+ StopDate time.Time
+ Workflow string
+ ResourceQty ExecutionRequirementsModel
}
class ScheduleInfo << (S,Aquamarine) >> {
+ Total int
+ NextExecutions []string
}
class SearchResult << (S,Aquamarine) >> {
+ Computing []ComputingModel
+ Datacenter []DatacenterModel
+ Storage []StorageModel
+ Data []DataModel
}
class StorageModel << (S,Aquamarine) >> {
+ ID string
- getRtype() rtype.Rtype
- getName() string
+ AddUserInput(inputs <font color=blue>map</font>[string]<font color=blue>interface</font>{})
}
class StorageNEWModel << (S,Aquamarine) >> {
+ Name string
+ Description string
+ ShortDescription string
+ Logo string
+ Type string
+ DCacronym string
+ URL string
+ Size uint
+ Encryption bool
+ Redundancy string
+ Throughput string
+ BookingPrice uint
}
class StorageObject << (S,Aquamarine) >> {
+ ReferenceID primitive.ObjectID
+ Inputs []string
+ Outputs []string
- getHost() *string
- getModel() (ResourceModel, error)
- setReference(rID primitive.ObjectID)
- getReference() primitive.ObjectID
- getRtype() rtype.Rtype
- getName() *string
- isLinked(rObjID string) LinkingState
- addLink(direction LinkingState, rObjID string)
}
class UserModel << (S,Aquamarine) >> {
+ ID string
+ Username string
+ Password string
+ Email string
}
class Workflow << (S,Aquamarine) >> {
+ Data <font color=blue>map</font>[string]DataObject
+ Computing <font color=blue>map</font>[string]ComputingObject
+ Storage <font color=blue>map</font>[string]StorageObject
+ Datacenter <font color=blue>map</font>[string]DatacenterObject
+ Schedules WorkflowSchedule
+ MxgraphXML string
+ GetExecutionRequirements(dcIDobj string) (ExecutionRequirementsModel, error)
+ GetResource(rObjID *string) ResourceObject
+ GetResourceMapByRtype(rt rtype.Rtype) <font color=blue>interface</font>{}
+ CreateResourceObject(rt rtype.Rtype) ResourceObject
+ AddObj(robj ResourceObject) *primitive.ObjectID
+ UpdateDB(userID string, workflowName string) error
+ UpdateObj(robj ResourceObject, objID string)
}
class WorkflowSchedule << (S,Aquamarine) >> {
+ IsService bool
+ StartDate time.Time
+ StopDate time.Time
+ Cron string
+ Duration uint
+ Events string
+ IsBooked bool
}
class Workspace << (S,Aquamarine) >> {
+ UserID string
+ Workflows <font color=blue>map</font>[string]Workflow
+ Data []string
+ Computing []string
+ Datacenter []string
+ Storage []string
- getRtype(rID string) rtype.Rtype
- updateDB() error
+ ConsumeMxGraphModel(xmlmodel MxGraphModel) (*Workflow, error, []error)
+ GetResources() <font color=blue>map</font>[rtype.Rtype][]string
+ GetWorkflow(workflowName string) *Workflow
+ GetWorkflows() []string
+ NewResource(rID string, rType string) error
+ GetAllWorkspacesProjects() <font color=blue>chan</font> *Workflow
}
class WorkspaceModel << (S,Aquamarine) >> {
+ UserID string
+ Data []DataModel
+ Computing []ComputingModel
+ Datacenter []DatacenterModel
+ Storage []StorageModel
}
class models.LinkingState << (T, #FF7700) >> {
}
class mxissue << (S,Aquamarine) >> {
- msg string
+ Error() string
}
}
"models.ComputingNEWModel" *-- "models.ComputingModel"
"models.DataNEWModel" *-- "models.DataModel"
"models.DatacenterNEWModel" *-- "models.DatacenterModel"
"models.StorageNEWModel" *-- "models.StorageModel"
"models.ResourceModel" <|-- "models.ComputingModel"
"models.ResourceObject" <|-- "models.ComputingObject"
"models.ResourceModel" <|-- "models.DataModel"
"models.ResourceObject" <|-- "models.DataObject"
"models.ResourceModel" <|-- "models.DatacenterModel"
"models.ResourceObject" <|-- "models.DatacenterObject"
"models.ResourceModel" <|-- "models.StorageModel"
"models.ResourceObject" <|-- "models.StorageObject"
"__builtin__.uint" #.. "models.LinkingState"
@enduml

View File

@@ -2,11 +2,12 @@
- [ ] In most of the components from 'models/' we have a method to add input and output to the model, however this linking of components is already done in oc-schedule when parsing the MxGraph. We need to determine if adding relations between components inside the objects themself is necessary.
- When running in debug mode with a breakpoint inside the first line of computing.addLink it is only called once
- [ ]
## MxGraph
- [ ] The ConsumeMxGraphModel is way too long, it should refactored and broken down in different sub methods
- mxcell are put inside an <object> tag when the settings have been opened, wether values have been set or not. Maybe we could find a way to make mxgraph add these whenever we add a component to the graph.
- then identify the links only
- [ ] It is unclear what are the inputs and the ouputs. It seems like they were implemented to link two components, but it seems redundant with the identification of links
- [ ] It is unclear what are the inputs and the ouputs. It seems like they were implemented to link two components, but it seems redundant with the identification of links
- This has been potentially tackled with the creation of a class to handle links between components. The components do no handle their own connections with other components, this task is delegated to the Link and Worlflow classes.

25
docs/linking_errors.md Normal file
View File

@@ -0,0 +1,25 @@
# Handling errors during workflows' post
Every time an user modify a worflow through oc-search's GUI its representation is sent through an XML to oc-catalog API.
To ensure a correct execution of the workflow we must look for irregularities in the workflow's design. Of course an error can be the result of a workflow being currently built by the user, with the corrective action coming.
This document aims at laying down all the requirements that each component must respect and show wether they have been implemented in the code or not.
## Computing
- [x] A computing component must be paired with a datacenter component
## Data
- [x] A data component must be linked to at least one computing component
## Datacenter
- [x] A datacenter component must be linked to at least one computing component
## Storage
- [x] A storage component must have at least one target or be the source of another component

View File

@@ -1,11 +1,13 @@
package models
import (
"encoding/xml"
"fmt"
"strings"
"cloud.o-forge.io/core/oc-catalog/models/rtype"
"cloud.o-forge.io/core/oc-catalog/services"
structtomap "github.com/Klathmon/StructToMap"
"github.com/beego/beego/v2/core/logs"
"go.mongodb.org/mongo-driver/bson/primitive"
)
@@ -35,21 +37,21 @@ type ComputingNEWModel struct {
ShortDescription string `json:"short_description,omitempty" required:"true" validate:"required"`
Logo string `json:"logo,omitempty" required:"true" validate:"required"`
// Type string `json:"type,omitempty" required:"true"`
Type string `json:"type,omitempty" required:"true"`
Owner string `json:"owner,omitempty"`
License string `json:"license,omitempty"`
Price uint `json:"price,omitempty"`
ExecutionRequirements ExecutionRequirementsModel `json:"execution_requirements,omitempty"`
// Dinputs []string `json:"dinputs,omitempty"` // Possibly redundant with Links object in oc-schedule
// Doutputs []string `json:"doutputs,omitempty"` // Possibly redundant with Links objects in oc-schedule
Dinputs []string `json:"dinputs,omitempty"` // Possibly redundant with Links object in oc-schedule
Doutputs []string `json:"doutputs,omitempty"` // Possibly redundant with Links objects in oc-schedule
Image string `json:"image,omitempty"`
Command string `json:"command,omitempty"`
Arguments []string `json:"arguments,omitempty"`
Environment []string `json:"environment,omitempty"`
// Ports []string `json:"ports,omitempty"`
Ports []string `json:"ports,omitempty"`
// CustomDeployment string `json:"custom_deployment,omitempty"`
@@ -75,10 +77,15 @@ func (model ComputingModel) getName() string {
type ComputingObject struct {
ReferenceID primitive.ObjectID `json:"referenceID" description:"Computing model ID"`
Inputs []string `json:"inputs"`
Outputs []string `json:"outputs"`
Inputs []string `json:"inputs"`
Outputs []string `json:"outputs"`
Image string `json:"image,omitempty"`
Command string `json:"command,omitempty"`
Arguments []string `json:"arguments,omitempty"`
Environment []string `json:"environment,omitempty"`
Ports []string `json:"ports,omitempty"`
DataCenterID string `json:"datacenterID" description:"Datacenter where the computing will be executed"`
DataCenterID string `json:"datacenterID" description:"Datacenter where the computing will be executed"`
}
func (obj ComputingObject) getHost() *string {
@@ -174,19 +181,45 @@ func PostOneComputing(obj ComputingNEWModel) (ID string, err error) {
return postOneResource(obj, rtype.COMPUTING)
}
func (obj ComputingModel) AddUserInput(inputs map[string]interface{} ){
func (obj *ComputingObject) AddUserInput(inputs []xml.Attr){
logs.Alert("AddUserInput() is going to throw some alerts while mxGraph GUI is not updated to adapt the inputs to the componant")
// So far only a few input to handle so a switch with a case for each type of attribute
// is enough, to prevent too much complexity
for key, value := range inputs {
switch strings.ToLower(key) {
for _, j := range(inputs){
setting, _ := structtomap.Convert(j)
// fmt.Println(strings.ToLower(setting["Name"]))
name := setting["Name"].(xml.Name).Local
value := setting["Value"]
switch name {
case "command":
obj.Command = value.(string)
case "arguments":
obj.Arguments = value.([]string)
case "args":
empty, sliced_arguments := getSliceSettings(value.(string))
if (!empty){
obj.Arguments = sliced_arguments
}
case "env" :
obj.Environment = value.([]string)
empty, sliced_arguments := getSliceSettings(value.(string))
if (!empty){
obj.Environment = sliced_arguments
}
default:
logs.Alert(fmt.Printf("%s is not an attribute of storage componants", key))
logs.Alert(fmt.Printf("%s is not an attribute of computing componants", name))
}
}
}
func getSliceSettings(string_to_parse string)(empty bool, sliced_string []string){
if len(string_to_parse) == 0 {
return true, nil
}
empty = false
sliced_string = strings.Split(string_to_parse," ")
return
}

View File

@@ -15,26 +15,13 @@ type DataNEWModel struct {
ShortDescription string `json:"short_description" required:"true" validate:"required"`
Logo string `json:"logo" required:"true" validate:"required"`
// Dtype string `json:"dtype"`
Dtype string `json:"dtype"`
Type string `json:"type,omitempty" required:"true" validate:"required" description:"Define type of data" example:"file"`
Example string `json:"example" required:"true" validate:"required" description:"base64 encoded data"`
Protocol []string `json:"protocol"` //TODO Enum type
Location string `json:"location" required:"true" validate:"required"`
}
type DataModel struct {
ID string `json:"ID" bson:"_id" required:"true" validate:"required"`
DataNEWModel `bson:",inline"`
}
func (obj DataModel) getRtype() rtype.Rtype {
return rtype.DATA
}
func (model DataModel) getName() string {
return model.Name
}
type DataIO struct {
Counter uint `description:"Incremental number starting from 0"`
}
@@ -43,6 +30,18 @@ type DataObject struct {
ReferenceID primitive.ObjectID `json:"referenceID" description:"Data model ID"`
}
type DataModel struct {
ID string `json:"ID" bson:"_id" required:"true" validate:"required"`
DataNEWModel `bson:",inline"`
}
func (obj DataModel) getRtype() rtype.Rtype {
return rtype.DATA
}
func (model DataModel) getName() string {
return model.Name
}
func (obj DataObject) getHost() *string {
return nil // Host is DC only attribute
}

View File

@@ -32,9 +32,9 @@ type DatacenterGpuModel struct {
type DatacenterNEWModel struct {
Name string `json:"name" required:"true"`
// Type string `json:"type,omitempty" required:"true"`
Type string `json:"type,omitempty" required:"true"`
Acronym string `json:"acronym" required:"true" description:"id of the DC"`
// Hosts []string `json:"hosts" required:"true" description:"list of host:port"`
Hosts []string `json:"hosts" required:"true" description:"list of host:port"`
Description string `json:"description" required:"true"`
ShortDescription string `json:"short_description" required:"true" validate:"required"`
Logo string `json:"logo" required:"true" validate:"required"`

55
models/links.go Normal file
View File

@@ -0,0 +1,55 @@
package models
import (
"cloud.o-forge.io/core/oc-catalog/models/rtype"
)
type Link struct {
// ID primitive.ObjectID `json:"ID" bson:"_id" required:"true" example:"5099803df3f4948bd2f98391"`
Source string `json:"source" description:"id in the workflow of the source object"`
Destination string `json:"destination" description:"id in the workflow of the destination object"`
DCLink bool `json:"dcLink" description:"is this a link with a datacenter"`
}
// Use ResourceObject parameter to process certain components type differently
// and Id's to identify each component as a node in an oriented graph
// In the case of DCLink we choose to always consider the DC as the destination
// in order to facilitate some logic
func NewLink(src ResourceObject, srcId string, dst ResourceObject, dstId string) (link Link) {
link.Source = srcId
link.Destination = dstId
// If the link is between a DC and a component make sure that the DC is destination
// and if the component is computing, update the DataCenterID
if (src.getRtype() == rtype.DATACENTER || dst.getRtype() == rtype.DATACENTER){
var linked ResourceObject
link.DCLink = true
if src.getRtype() == rtype.DATACENTER {
linked = dst
} else {
linked = src
}
if( link.DCLink && src.getRtype() == rtype.DATACENTER){
link.Destination = srcId
link.Source = dstId
}
if (linked.getRtype() == rtype.COMPUTING){
linked.(*ComputingObject).DataCenterID = link.Destination
}
}
return
}
// So far only computing components expect the ID of the DC in their attributes
// func (l *Link) AddLinkToDataCenter(component models.ComputingModel) {
// }

View File

@@ -8,18 +8,68 @@ type MxGraphModel struct {
XMLName xml.Name `xml:"mxGraphModel"`
Root struct {
XMLName xml.Name `xml:"root"`
MxCell []MxCell `xml:"mxCell"`
XMLName xml.Name `xml:"root"`
MxCell []MxCell `xml:"mxCell"`
MxObject *[]MxObject `xml:"object"`
MxLink []MxLink
}
}
type MxCell struct {
XMLName xml.Name `xml:"mxCell"`
ID string `xml:"id,attr"`
Parent *string `xml:"parent,attr"`
RID *string `xml:"rID,attr"`
Source *string `xml:"source,attr"`
Target *string `xml:"target,attr"`
XMLName xml.Name `xml:"mxCell"`
ID string `xml:"id,attr"`
RID *string `xml:"rID,attr"`
Rtype string `xml:"rType,attr"`
Parent *string `xml:"parent,attr"`
Edge *string `xml:"edge,attr"`
Source *string `xml:"source,attr"`
Target *string `xml:"target,attr"`
}
type MxLink struct {
ID string `xml:"id,attr"`
Source string `xml:"source,attr"`
Target string `xml:"target,attr"`
}
type MxObject struct {
XMLName xml.Name `xml:"object"`
ID string `xml:"id,attr"`
Settings []xml.Attr `xml:",any,attr"`
MxCell MxCell `xml:"mxCell"`
}
// Didn't manage to differentiate Links and cells containing components using
// only structures and unmarshal, so we use this method post-umarshalling
func (g *MxGraphModel) createLinks() {
var cells_without_links []MxCell
for i, mxcell := range g.Root.MxCell {
if mxcell.Edge != nil {
mxcell.processLinks()
newLink := MxLink{mxcell.ID,*mxcell.Source,*mxcell.Target}
g.Root.MxLink = append(g.Root.MxLink,newLink)
} else {
cells_without_links = append(cells_without_links,g.Root.MxCell[i])
}
}
g.Root.MxCell = nil
g.Root.MxCell = cells_without_links
}
func (cell *MxCell) processLinks() {
v := ""
if cell.Source == nil {
cell.Source = &v
}
if cell.Target == nil {
cell.Target = &v
}
}
type mxissue struct {
@@ -33,3 +83,4 @@ func (m *mxissue) Error() string {
func newMxIssue(message string) error {
return &mxissue{message}
}

View File

@@ -4,6 +4,8 @@ import (
"context"
"encoding/xml"
"errors"
"net/url"
"os"
"sort"
"time"
@@ -50,16 +52,19 @@ const SchedulesDB = "schedules"
type Workflow struct {
// The key of the map is the ID of the object itself
Data map[string]DataObject `json:"data"`
Computing map[string]ComputingObject `json:"computing"`
Storage map[string]StorageObject `json:"storage"`
Datacenter map[string]DatacenterObject `json:"datacenter"` //TODO: Decide if there should be multiple objects of a datacenter
Data map[string]DataObject `json:"data"`
Computing map[string]ComputingObject `json:"computing"`
Storage map[string]StorageObject `json:"storage"`
Datacenter map[string]DatacenterObject `json:"datacenter"` //TODO: Decide if there should be multiple objects of a datacenter
Links map[string]Link `json:"link"`
Schedules WorkflowSchedule `json:"schedules"`
MxgraphXML string `description:"State of the mxgraph"`
}
// TODO : describe what use case this interface satisfies
type ResourceObject interface {
getHost() *string
getName() *string
@@ -71,18 +76,25 @@ type ResourceObject interface {
addLink(direction LinkingState, rObjID string)
}
// This type allows to process computing and storage component
// which can get input from the user
type EditableResourceObject interface{
ResourceObject
addUserInput(map[string]interface{})
}
// Get a sum of all execution requirements attached to a DC obj
func (w Workflow) GetExecutionRequirements(dcIDobj string) (ret ExecutionRequirementsModel, err error) {
func (wf Workflow) GetExecutionRequirements(dcIDobj string) (ret ExecutionRequirementsModel, err error) {
// Find the id of the DC obj
if _, ok := w.Datacenter[dcIDobj]; !ok {
if _, ok := wf.Datacenter[dcIDobj]; !ok {
return ExecutionRequirementsModel{}, errors.New("DC obj" + dcIDobj + " doesn't exist in the Workflow")
}
// Get all elements that are attached to the DC
for _, computingObj := range w.Computing {
for _, computingObj := range wf.Computing {
if computingObj.DataCenterID == dcIDobj {
mymodel, err := computingObj.getModel()
if err != nil {
@@ -159,6 +171,7 @@ func (w *Workflow) CreateResourceObject(rt rtype.Rtype) ResourceObject {
default:
res = nil
}
return res
}
@@ -169,6 +182,13 @@ func (w *Workflow) AddObj(robj ResourceObject) *primitive.ObjectID {
return &outputID
}
func (w *Workflow) AddLinkToWorkflow (link Link, id string){
if w.Links == nil {
w.Links = make(map[string]Link)
}
w.Links[id] = link
}
func (w *Workflow) UpdateDB(userID, workflowName string) error {
_, err := services.MngoCollWorkspace.UpdateOne(services.MngoCtx,
@@ -510,6 +530,14 @@ func ParseMxGraph(username, workflowName, xmlData string) (err error, mxissues [
//return errors.New("Can't modify a booked workflow"), nil
}
decodedValue, err := url.QueryUnescape(xmlData)
if err != nil {
return err, nil
}
// TEMPORARY test the xml created
os.WriteFile("graph.xml", []byte(decodedValue), 0660)
var xmlModel MxGraphModel
// logs.Debug(xmlData)
@@ -518,13 +546,16 @@ func ParseMxGraph(username, workflowName, xmlData string) (err error, mxissues [
return err, nil
}
xmlModel.createLinks()
targetWorkspaceWorkflow, err, mxissues := userWorkspace.ConsumeMxGraphModel(xmlModel)
if err != nil {
return err, nil
}
targetWorkspaceWorkflow.MxgraphXML = xmlData
targetWorkspaceWorkflow.Schedules = currentWorkflow.Schedules //TODO: Probably we should move schudles outside the workflow
targetWorkspaceWorkflow.Schedules = currentWorkflow.Schedules //TODO: Probably we should move schedules outside the workflow
_, err = services.MngoCollWorkspace.UpdateOne(services.MngoCtx,
primitive.M{"_id": username},
@@ -564,7 +595,7 @@ func FindSliceInSlice(slice1 []string, slice2 []string) (int, int, bool) {
return -1, -1, false
}
func (w Workspace) ConsumeMxGraphModel(xmlmodel MxGraphModel) (returned_wf *Workflow, err error, issues []error) {
func (ws Workspace) ConsumeMxGraphModel(xmlmodel MxGraphModel) (returned_wf *Workflow, err error, issues []error) {
returned_wf = &Workflow{}
@@ -574,42 +605,41 @@ func (w Workspace) ConsumeMxGraphModel(xmlmodel MxGraphModel) (returned_wf *Work
return xmlmodel.Root.MxCell[i].RID != nil
})
// For each cell of the xml graph,
// in the case cell has a rID retrieve its rType from the value of rID of the componant in the worfklow
// retrieve the componant's type
// create an object from the rType
// update the existing workflow with the new componant
// or by defautlt : the cell represents an arrow
// if the source or the target of the arrow is a datacenter
// define which end of the arrow is the DC
// if the other other end of the arrow is a computing component
// create a computing object
// attach the DC to it
// update the workflow with the object : create the list of this type of component or update the list with the id of the component with the object
// Create the object and add it to the appropriate list
// for all the components with setting, which are identified
// by a MxObject tag in the xml
for _, object := range *xmlmodel.Root.MxObject{
resObj, err, mxissues := returned_wf.mxCellToComponent(object.MxCell,ws)
if err != nil {
issues = append(issues, mxissues...)
}
// add the component to the worflow's attribute that stores
// all components in a map[string]Component where the key
// is the component's ID in the mxGraph and the value the Component object
returned_wf.UpdateObj(resObj,object.ID)
// Construct the object corresponding to the componant's type and use its addUserInput method
if(resObj.getRtype() == rtype.COMPUTING){
comp_obj := returned_wf.GetResource(&object.ID).(*ComputingObject)
comp_obj.AddUserInput(object.Settings)
returned_wf.UpdateObj(comp_obj,object.ID)
}
// if(resObj.getRtype() == rtype.DATA){
// }
}
for _, cell := range xmlmodel.Root.MxCell {
switch {
case cell.RID != nil:
// Case of a Resource
rType := w.getRtype(*cell.RID)
if rType == rtype.INVALID {
return nil,
errors.New("Refering to a rID that is not in the workflow"),
nil
}
// Generate ObjectID for the reference ID
rIDObj, err := primitive.ObjectIDFromHex(*cell.RID)
resObj, err, mxissues := returned_wf.mxCellToComponent(cell,ws)
if err != nil {
return nil,
errors.New("Bad ID format: " + *cell.RID),
nil
issues = append(issues, mxissues...)
}
resObj := returned_wf.CreateResourceObject(rType)
resObj.setReference(rIDObj)
returned_wf.UpdateObj(resObj, cell.ID)
case cell.ID == "0" || cell.ID == "1":
@@ -635,93 +665,72 @@ func (w Workspace) ConsumeMxGraphModel(xmlmodel MxGraphModel) (returned_wf *Work
continue
}
if sourceObj.getRtype() == rtype.DATACENTER || targetObj.getRtype() == rtype.DATACENTER {
var datacenter, datacenterLinked *string
if sourceObj.getRtype() == rtype.DATACENTER {
datacenter = cell.Source
datacenterLinked = cell.Target
} else {
datacenter = cell.Target
datacenterLinked = cell.Source
}
switch returned_wf.GetResource(datacenterLinked).getRtype() {
case rtype.COMPUTING:
computingObj := returned_wf.GetResource(datacenterLinked).(*ComputingObject)
// We should always get a ID because we already registered resources and discarded which doesn't correspond to existent models
computingObj.DataCenterID = *datacenter
returned_wf.UpdateObj(computingObj, *datacenterLinked)
}
} else {
targetObj.addLink(INPUT, *cell.Source)
returned_wf.UpdateObj(targetObj, *cell.Target) // save back
// If we have a relationship of:
// Source ----> Target
//
// The Source will be in the INPUTs of the Target.
// But we also must make sure that the Target will be in the OUTPUTs of the Source
sourceObj.addLink(OUTPUT, *cell.Target)
returned_wf.UpdateObj(sourceObj, *cell.Source)
}
// Not root nor resource. Should be only links
// If is a invalid link, we can't save it in the DB
// We should always get a ID because we already registered resources and discarded which doesn't correspond to existent models
// save back
// If we have a relationship of:
// Source ----> Target
//
// The Source will be in the INPUTs of the Target.
// But we also must make sure that the Target will be in the OUTPUTs of the Source
}
}
dcslist := make(map[string]bool)
dataslist := make(map[string]bool)
// datalist := make(map[string]bool)
issues = returned_wf.CreateLinks(xmlmodel.Root.MxLink, issues)
issues = returned_wf.CheckLinks(issues)
// dcslist := make(map[string]bool)
// dataslist := make(map[string]bool)
// // datalist := make(map[string]bool)
// Test wether the computing componants are linked with a DC
for _, comp := range returned_wf.Computing {
if comp.DataCenterID == "" {
issues = append(issues, errors.New("Computing "+*comp.getName()+" without a Datacenter"))
} else {
// If doesn't exist in the list, means is new element to register as used
dcslist[comp.DataCenterID] = true
// // Test wether the computing components are linked with a DC
// for _, comp := range returned_wf.Computing {
// if comp.DataCenterID == "" {
// issues = append(issues, errors.New("Computing "+*comp.getName()+" without a Datacenter"))
// } else {
// // If doesn't exist in the list, means is new element to register as used
// dcslist[comp.DataCenterID] = true
}
// }
for _, dcin := range comp.Inputs {
switch returned_wf.GetResource(&dcin).getRtype() {
case rtype.DATA:
dataslist[dcin] = true
}
}
// for _, dcin := range comp.Inputs {
// switch returned_wf.GetResource(&dcin).getRtype() {
// case rtype.DATA:
// dataslist[dcin] = true
// }
// }
for _, dcout := range comp.Outputs {
switch returned_wf.GetResource(&dcout).getRtype() {
case rtype.DATA:
dataslist[dcout] = true
}
}
// for _, dcout := range comp.Outputs {
// switch returned_wf.GetResource(&dcout).getRtype() {
// case rtype.DATA:
// dataslist[dcout] = true
// }
// }
}
// }
for _, storage_component := range returned_wf.Storage {
if storage_component.Inputs == nil && storage_component.Outputs == nil {
issues = append(issues, errors.New("Storage "+*storage_component.getName()+" without compatible inputs and outputs"))
}
}
// for _, storage_component := range returned_wf.Storage {
// if storage_component.Inputs == nil && storage_component.Outputs == nil {
// issues = append(issues, errors.New("Storage "+*storage_component.getName()+" without compatible inputs and outputs"))
// }
// }
for dcID, dc_component := range returned_wf.Datacenter {
// if rID doesn't exist in the list, it means that it's not used
if _, ok := dcslist[dcID]; !ok {
issues = append(issues, errors.New("DC "+*dc_component.getName()+" not atached to any Computing"))
}
}
// for dcID, dc_component := range returned_wf.Datacenter {
// // if rID doesn't exist in the list, it means that it's not used
// if _, ok := dcslist[dcID]; !ok {
// issues = append(issues, errors.New("DC "+*dc_component.getName()+" not attached to any Computing"))
// }
// }
for dcID, data_component := range returned_wf.Data {
// if rID doesn't exist in the list, it means that it's not used
if _, ok := dataslist[dcID]; !ok {
issues = append(issues, errors.New("Data "+*data_component.getName()+" not atached to any Computing"))
}
}
// for dcID, data_component := range returned_wf.Data {
// // if rID doesn't exist in the list, it means that it's not used
// if _, ok := dataslist[dcID]; !ok {
// issues = append(issues, errors.New("Data "+*data_component.getName()+" not attached to any Computing"))
// }
// }
//////////////////////////////////////////////////////////
// //
@@ -824,6 +833,116 @@ func (w Workspace) ConsumeMxGraphModel(xmlmodel MxGraphModel) (returned_wf *Work
return
}
func (w *Workflow) CreateLinks(links []MxLink, issues []error) []error {
for _, link := range links {
if (len(link.Source) > 0 && len(link.Target) > 0){
sourceObj := w.GetResource(&link.Source)
targetObj := w.GetResource(&link.Target)
link_object := NewLink(sourceObj,link.Source, targetObj, link.Target)
w.AddLinkToWorkflow(link_object,link.ID)
} else {
issues = append(issues, w.processLinkErrors(link))
}
}
return issues
}
func (w *Workflow) processLinkErrors(link MxLink) (issue error) {
if len(link.Source) == 0 && len(link.Target) == 0 {
issue = errors.New("Arrow "+link.ID+" is alone")
} else if len(link.Source) == 0{
targetObj := w.GetResource(&link.Target)
issue = errors.New("Arrow ("+link.ID+") to "+*targetObj.getName()+" without parent")
} else {
sourceObj := w.GetResource(&link.Source)
issue = errors.New("Arrow "+link.ID+" from "+*sourceObj.getName()+" without target")
}
return issue
}
func (w *Workflow) CheckLinks(issues []error) []error {
// Check that storage components have a valid link
for id, storage := range w.Storage {
if(!w.IsComponentSrc(id) && !w.IsComponentDst(id)){
issues = append(issues, errors.New("Storage "+*storage.getName()+" without compatible inputs and outputs"))
}
}
// Check that data components are linked to a computing component
for id, data := range w.Data {
if(!w.HasLinkageToComputing(id)){
issues = append(issues, errors.New("Data "+*data.getName()+" not attached to any Computing"))
}
}
// Check that DC is linked to a computing component
for id, dc:= range w.Datacenter {
if(!w.HasLinkageToComputing(id)){
issues = append(issues, errors.New("Datacenter "+*dc.getName()+" not attached to any Computing"))
}
}
// Check that all data computing components are linked to a DC
for id,comp:= range w.Computing {
if(!w.HasLinkageToDC(id)){
issues = append(issues, errors.New("Computing "+*comp.getName()+" not attached to any datacenter"))
}
}
return issues
}
func (w *Workflow) IsComponentSrc(id string) bool {
for _, link := range w.Links{
if(link.Source == id && link.Source != ""){
return true
}
}
return false
}
func (w *Workflow) IsComponentDst(id string) bool {
for _, link := range w.Links{
if(link.Destination == id && link.Source != ""){
return true
}
}
return false
}
func (w *Workflow) HasLinkageToComputing(id string) bool {
for idComputing, _ := range w.Computing {
if( (w.IsComponentSrc(id) && w.IsComponentDst(idComputing)) || (w.IsComponentSrc(idComputing) && w.IsComponentDst(id))){
return true
}
}
return false
}
func (w *Workflow) HasLinkageToDC(id string) bool {
for _, link := range w.Links{
if(link.Source == id && link.DCLink){
return true
}
}
return false
}
func sumExecutionReqs(exeqReq ...ExecutionRequirementsModel) (ret ExecutionRequirementsModel) {
for _, v := range exeqReq {
ret.CPUs += v.CPUs
@@ -1005,9 +1124,49 @@ func CheckAndBookWorkflowSchedule(username, workflowName string, book bool) (myR
SchedulesDB: &currentWorkflow.Schedules}},
)
if err != nil {
if err != nil {
logs.Critical("Internal error when updating in DB: " + err.Error())
}
return myRet, nil
}
// Not sure if this method handles error propagation well
func (wf Workflow) mxCellToComponent(cell MxCell, ws Workspace) (resObj ResourceObject,err error, issues []error){
rType := ws.getRtype(*cell.RID)
if rType == rtype.INVALID {
return nil,
errors.New("Refering to a rID that is not in the workflow"),
nil
}
// Generate ObjectID for the reference ID
rIDObj, err := primitive.ObjectIDFromHex(*cell.RID)
if err != nil {
return nil,
errors.New("Bad ID format: " + *cell.RID),
nil
}
resObj = wf.CreateResourceObject(rType)
resObj.setReference(rIDObj)
return
}
// func (ws Workspace) extractMxCell(xmlModel MxGraphModel){
// // Iterate through all objects of the MxGraph
// graphObjects := xmlModel.Root.MxObject
// for _, object := range(*graphObjects){
// current_obj_id, _ := strconv.Atoi(object.ID)
// inside_cell_id := strconv.Itoa(current_obj_id + 1)
// cell := ws.GetResource(&inside_cell_id)
// // component := w.GetResource(cell.RID)
// fmt.Print(cell)
// }
// // Extract the mxCell object
// // Invoke the addParameter method from the component
// // Edit the ID to get the object's one
// }

Binary file not shown.

After

Width:  |  Height:  |  Size: 345 KiB