core/deprecated-oc-catalog
5
0
Fork 0
Code Issues 5 Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: core:main
Branches Tags
core:main
core:feature/sort_links
core:argo_workflow
core:feature/links_model
core:bugfix/settings_handling
core:bug/api-problem-workflows
core:pierre_configuration
..
compare: core:bugfix/settings_handling
Branches Tags
core:main
core:feature/sort_links
core:argo_workflow
core:feature/links_model
core:bugfix/settings_handling
core:bug/api-problem-workflows
core:pierre_configuration

No commits in common. "main" and "bugfix/settings_handling" have entirely different histories.
main ... bugfix/set

39 changed files with 215 additions and 552 deletions
Show Stats Expand all files Collapse all files

4
Dockerfile
View File

@ -13,8 +13,8 @@ RUN bee generate routers
# Generating the swagger
RUN timeout 20 bee run -gendoc=true -downdoc=true -runmode=dev || :
RUN sed -i 's/http:\/\/127.0.0.1:8080\/swagger\/swagger.json/swagger.json/g' swagger/index.html
RUN sed -i 's/https:\/\/petstore.swagger.io\/v2\/swagger.json/swagger.json/g' swagger/index.html
RUN sed -i 's/http:\/\/127.0.0.1:8080\/swagger\/swagger.json/swagger.json/g' swagger/index.html &&\
sed -i 's/https:\/\/petstore.swagger.io\/v2\/swagger.json/swagger.json/g' swagger/index.html
RUN ls -l routers

31
README.md
View File

@ -31,30 +31,6 @@ const ui = SwaggerUIBundle({
This issue is fixed in the Dockerfile build with two `sed` to change the value of the swagger file URL.
## Deployment with Docker compose
## Running the app locally
In order to run the application we need to correct some errors that beego generates.
**Router generation**
Beego generates the routers from the comment of the controllers methods. However, there seems to be some errors when we let beego generate th routers from `bee run`.
We need to use `bee generate routers` to have a working router file.
**Swagger generation**
Using `bee run -downdoc=true -gendoc=true` beego download a swagger template and generate the JSON file (swagger.json) in `swagger/`. However the swagger template in `swagger/index.html` does not change the exemple's url for our current JSON file. When running locally we have to edit the call to construct the object holding the swagger information, with the parameter url set with the relative path to our JSON swagger file :
```
const ui = SwaggerUIBundle({
url: "swagger.json",
...
...
});
```
This issue is fixed in the Dockerfile build with two `sed` to change the value of the swagger file URL.
## Deployment with Docker compose
Deploy with docker:
@ -73,18 +49,15 @@ or out of docker `./scripts/populate_models.sh ./scripts/demo.json`
### Multinode
Deploy :
### Multinode
Deploy :
`docker-compose -f docker-compose.yml -f docker-compose.backend.yml -f docker-compose.multi.yml up --build`
## Populating the MongoDB database
On the machine running the mongoDB container, from the root of the project run this line :
From the root of the projet run :
`./scripts/populate_models.sh ./scripts/demo.json`
`./scripts/multinode.sh ./scripts/demo.json`
This script should be updated to be ran from anywhere.

2
controllers/computing.go
View File

@ -1,7 +1,7 @@
package controllers
import (
"cloud.o-forge.io/core/deprecated-oc-catalog/models"
"cloud.o-forge.io/core/oc-catalog/models"
beego "github.com/beego/beego/v2/server/web"
)

2
controllers/data.go
View File

@ -1,7 +1,7 @@
package controllers
import (
"cloud.o-forge.io/core/deprecated-oc-catalog/models"
"cloud.o-forge.io/core/oc-catalog/models"
beego "github.com/beego/beego/v2/server/web"
"github.com/go-playground/validator/v10"

2
controllers/datacenter.go
View File

@ -1,7 +1,7 @@
package controllers
import (
"cloud.o-forge.io/core/deprecated-oc-catalog/models"
"cloud.o-forge.io/core/oc-catalog/models"
beego "github.com/beego/beego/v2/server/web"
"github.com/go-playground/validator/v10"

2
controllers/schedule.go
View File

@ -3,7 +3,7 @@ package controllers
import (
"time"
"cloud.o-forge.io/core/deprecated-oc-catalog/models"
"cloud.o-forge.io/core/oc-catalog/models"
"github.com/beego/beego/v2/core/logs"
beego "github.com/beego/beego/v2/server/web"
)

2
controllers/search.go
View File

@ -1,7 +1,7 @@
package controllers
import (
"cloud.o-forge.io/core/deprecated-oc-catalog/models"
"cloud.o-forge.io/core/oc-catalog/models"
beego "github.com/beego/beego/v2/server/web"
)

2
controllers/storage.go
View File

@ -1,7 +1,7 @@
package controllers
import (
"cloud.o-forge.io/core/deprecated-oc-catalog/models"
"cloud.o-forge.io/core/oc-catalog/models"
beego "github.com/beego/beego/v2/server/web"
)

2
controllers/user.go
View File

@ -1,7 +1,7 @@
package controllers
import (
"cloud.o-forge.io/core/deprecated-oc-catalog/models"
"cloud.o-forge.io/core/oc-catalog/models"
beego "github.com/beego/beego/v2/server/web"
)

2
controllers/workflow.go
View File

@ -3,7 +3,7 @@ package controllers
import (
"time"
"cloud.o-forge.io/core/deprecated-oc-catalog/models"
"cloud.o-forge.io/core/oc-catalog/models"
"github.com/beego/beego/v2/core/logs"
beego "github.com/beego/beego/v2/server/web"
"github.com/vk496/cron"

2
controllers/workspace.go
View File

@ -1,7 +1,7 @@
package controllers
import (
"cloud.o-forge.io/core/deprecated-oc-catalog/models"
"cloud.o-forge.io/core/oc-catalog/models"
beego "github.com/beego/beego/v2/server/web"
)

47
docs/UML/diag_class_links.puml
View File

@ -1,47 +0,0 @@
@startuml class_links
class LinkGraph {
+ []Link links
+ void AddLinkToGraph(Link link)
+ boolean,str HasPrevious(Link)
+ boolean,str HasNext(Link)
}
class Link {
+ str source
+ str destination
+ boolean DCLink
+ *Link NewLink(interface{} src, interface{} dst)
+ void AddLinkToDataCenter()
}
note left of LinkGraph::HasPrevious
checks if the component whose ID is in src is the dst of
any other Link of the list on Link
end note
note top of Link
Links need to be redefined in the sense that they are currently used
both :
- to connect a component to a DC
- to represent the interractions between components
end note
note right of Link::DCLink
set at construction of the object and used to order the links
end note
note left of Link::NewLink
Must test if the parameters check the type constraints
and raise errors for the GUI if necessary
end note
LinkGraph*--"links"Link
@enduml

54
docs/UML/mxgraph.puml
View File

@ -1,54 +0,0 @@
@startuml mxgraph
package MxGraph {
class MxGraphModel {
- XMLName xml.Name
+ Root : RootStruct
+ createLinks()
}
class RootStruct {
- XMLName xml.Name
- MxCell[] MxCell
- MxObject[] MxObject
- MxLink[] MxLink
}
class MxCell {
- XMLName xml.Name
+ ID string
+ RID *string
+ Rtype string
+ Parent *string
+ Edge *string
+ Source *string
+ Target *string
+ processLinks()
}
class MxLink {
+ ID string
+ Source string
+ Target string
}
class MxObject {
- XMLName xml.Name
+ ID string
+ Settings []xml.Attr
+ MxCell MxCell
}
class mxissue {
- msg string
+ Error() string
}
MxGraphModel *-- RootStruct : Root
RootStruct *-- MxCell : MxCell
RootStruct *-- MxObject : MxObject
RootStruct *-- MxLink : MxLink
MxObject *-- MxCell : MxCell
}
@enduml

2
docs/UML/plantuml/diagram_object_routers.puml
View File

@ -1,2 +0,0 @@
@startuml
@enduml

5
docs/graph_construction.md
View File

@ -1,5 +0,0 @@
For an easier parsing and constrution of dependencies we decided that :
- Computing elements must be linked between one another, to represent the chaining of events
- A storage component can be shared by several computing component, the direction of the arrow will determine if it's for writting or reading

5
docs/identified_problems.md
View File

@ -2,12 +2,11 @@
- [ ] In most of the components from 'models/' we have a method to add input and output to the model, however this linking of components is already done in oc-schedule when parsing the MxGraph. We need to determine if adding relations between components inside the objects themself is necessary.
- When running in debug mode with a breakpoint inside the first line of computing.addLink it is only called once
- [ ]
## MxGraph
- [ ] The ConsumeMxGraphModel is way too long, it should refactored and broken down in different sub methods
- mxcell are put inside an <object> tag when the settings have been opened, wether values have been set or not. Maybe we could find a way to make mxgraph add these whenever we add a component to the graph.
- then identify the links only
- [ ] It is unclear what are the inputs and the ouputs. It seems like they were implemented to link two components, but it seems redundant with the identification of links
- This has been potentially tackled with the creation of a class to handle links between components. The components do no handle their own connections with other components, this task is delegated to the Link and Worlflow classes.
- [ ] It is unclear what are the inputs and the ouputs. It seems like they were implemented to link two components, but it seems redundant with the identification of links

25
docs/linking_errors.md
View File

@ -1,25 +0,0 @@
# Handling errors during workflows' post
Every time an user modify a worflow through oc-search's GUI its representation is sent through an XML to oc-catalog API.
To ensure a correct execution of the workflow we must look for irregularities in the workflow's design. Of course an error can be the result of a workflow being currently built by the user, with the corrective action coming.
This document aims at laying down all the requirements that each component must respect and show wether they have been implemented in the code or not.
## Computing
- [x] A computing component must be paired with a datacenter component
## Data
- [x] A data component must be linked to at least one computing component
## Datacenter
- [x] A datacenter component must be linked to at least one computing component
## Storage
- [x] A storage component must have at least one target or be the source of another component

3
go.mod
View File

@ -1,11 +1,10 @@
module cloud.o-forge.io/core/deprecated-oc-catalog
module cloud.o-forge.io/core/oc-catalog
go 1.15
require github.com/beego/beego/v2 v2.0.1
require (
github.com/Klathmon/StructToMap v0.0.0-20140724123129-3d0229e2dce7
github.com/antihax/optional v1.0.0
github.com/aws/aws-sdk-go v1.36.29 // indirect
github.com/beego/beego v1.12.12

2
go.sum
View File

@ -1,8 +1,6 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Klathmon/StructToMap v0.0.0-20140724123129-3d0229e2dce7 h1:n0MD6UkwbgGHtXsmfgVzC2+ZbHzIsScpbq9ZGI18074=
github.com/Klathmon/StructToMap v0.0.0-20140724123129-3d0229e2dce7/go.mod h1:xdrQDwHlKUmv8yiElMx6W0W10cLkqpeSEUUib8KGtv4=
github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=

6
main.go
View File

@ -3,9 +3,9 @@ package main
import (
"os"
"cloud.o-forge.io/core/deprecated-oc-catalog/conf"
"cloud.o-forge.io/core/deprecated-oc-catalog/routers"
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
"cloud.o-forge.io/core/oc-catalog/conf"
"cloud.o-forge.io/core/oc-catalog/routers"
"cloud.o-forge.io/core/oc-catalog/services"
"github.com/beego/beego/logs"
beego "github.com/beego/beego/v2/server/web"

10
models/computing.go
View File

@ -5,8 +5,8 @@ import (
"fmt"
"strings"
"cloud.o-forge.io/core/deprecated-oc-catalog/models/rtype"
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
"cloud.o-forge.io/core/oc-catalog/models/rtype"
"cloud.o-forge.io/core/oc-catalog/services"
structtomap "github.com/Klathmon/StructToMap"
"github.com/beego/beego/v2/core/logs"
"go.mongodb.org/mongo-driver/bson/primitive"
@ -77,15 +77,15 @@ func (model ComputingModel) getName() string {
type ComputingObject struct {
ReferenceID primitive.ObjectID `json:"referenceID" description:"Computing model ID"`
Inputs []string `json:"inputs"`
Outputs []string `json:"outputs"`
Inputs []string `json:"inputs"`
Outputs []string `json:"outputs"`
Image string `json:"image,omitempty"`
Command string `json:"command,omitempty"`
Arguments []string `json:"arguments,omitempty"`
Environment []string `json:"environment,omitempty"`
Ports []string `json:"ports,omitempty"`
DataCenterID string `json:"datacenterID" description:"Datacenter where the computing will be executed"`
DataCenterID string `json:"datacenterID" description:"Datacenter where the computing will be executed"`
}
func (obj ComputingObject) getHost() *string {

4
models/data.go
View File

@ -1,8 +1,8 @@
package models
import (
"cloud.o-forge.io/core/deprecated-oc-catalog/models/rtype"
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
"cloud.o-forge.io/core/oc-catalog/models/rtype"
"cloud.o-forge.io/core/oc-catalog/services"
"github.com/beego/beego/v2/core/logs"
"go.mongodb.org/mongo-driver/bson/primitive"
)

4
models/datacenter.go
View File

@ -4,8 +4,8 @@ import (
"net"
"time"
"cloud.o-forge.io/core/deprecated-oc-catalog/models/rtype"
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
"cloud.o-forge.io/core/oc-catalog/models/rtype"
"cloud.o-forge.io/core/oc-catalog/services"
"github.com/beego/beego/v2/core/logs"
"go.mongodb.org/mongo-driver/bson/primitive"
)

4
models/generic.go
View File

@ -3,8 +3,8 @@ package models
import (
"errors"
"cloud.o-forge.io/core/deprecated-oc-catalog/models/rtype"
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
"cloud.o-forge.io/core/oc-catalog/models/rtype"
"cloud.o-forge.io/core/oc-catalog/services"
"github.com/beego/beego/v2/core/logs"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"

53
models/links.go
View File

@ -1,53 +0,0 @@
package models
import (
"cloud.o-forge.io/core/deprecated-oc-catalog/models/rtype"
)
type Link struct {
// ID primitive.ObjectID `json:"ID" bson:"_id" required:"true" example:"5099803df3f4948bd2f98391"`
Source string `json:"source" description:"id in the workflow of the source object"`
Destination string `json:"destination" description:"id in the workflow of the destination object"`
DCLink bool `json:"dcLink" description:"is this a link with a datacenter"`
}
// Use ResourceObject parameter to process certain components type differently
// and Id's to identify each component as a node in an oriented graph
// In the case of DCLink we choose to always consider the DC as the destination
// in order to facilitate some logic
func NewLink(src ResourceObject, srcId string, dst ResourceObject, dstId string) (link Link) {
link.Source = srcId
link.Destination = dstId
// If the link is between a DC and a component make sure that the DC is destination
// and if the component is computing, update the DataCenterID
if (src.getRtype() == rtype.DATACENTER || dst.getRtype() == rtype.DATACENTER){
var linked ResourceObject
link.DCLink = true
if src.getRtype() == rtype.DATACENTER {
linked = dst
} else {
linked = src
}
if( link.DCLink && src.getRtype() == rtype.DATACENTER){
link.Destination = srcId
link.Source = dstId
}
if (linked.getRtype() == rtype.COMPUTING){
linked.(*ComputingObject).DataCenterID = link.Destination
}
}
return
}

58
models/mxgraph.go
View File

@ -8,29 +8,22 @@ type MxGraphModel struct {
XMLName xml.Name `xml:"mxGraphModel"`
Root struct {
XMLName xml.Name `xml:"root"`
MxCell []MxCell `xml:"mxCell"`
XMLName xml.Name `xml:"root"`
MxCell []MxCell `xml:"mxCell"`
MxObject *[]MxObject `xml:"object"`
MxLink []MxLink
}
}
type MxCell struct {
XMLName xml.Name `xml:"mxCell"`
ID string `xml:"id,attr"`
RID *string `xml:"rID,attr"`
Rtype string `xml:"rType,attr"`
Parent *string `xml:"parent,attr"`
Edge *string `xml:"edge,attr"`
Source *string `xml:"source,attr"`
Target *string `xml:"target,attr"`
XMLName xml.Name `xml:"mxCell"`
ID string `xml:"id,attr"`
Parent *string `xml:"parent,attr"`
RID *string `xml:"rID,attr"`
Source *string `xml:"source,attr"`
Target *string `xml:"target,attr"`
Rtype *string `xml:"rType,attr"`
}
type MxLink struct {
ID string `xml:"id,attr"`
Source string `xml:"source,attr"`
Target string `xml:"target,attr"`
}
type MxObject struct {
XMLName xml.Name `xml:"object"`
@ -39,39 +32,6 @@ type MxObject struct {
MxCell MxCell `xml:"mxCell"`
}
// Didn't manage to differentiate Links and cells containing components using
// only structures and unmarshal, so we use this method post-umarshalling
func (g *MxGraphModel) createLinks() {
var cells_without_links []MxCell
for i, mxcell := range g.Root.MxCell {
if mxcell.Edge != nil {
mxcell.processLinks()
newLink := MxLink{mxcell.ID,*mxcell.Source,*mxcell.Target}
g.Root.MxLink = append(g.Root.MxLink,newLink)
} else {
cells_without_links = append(cells_without_links,g.Root.MxCell[i])
}
}
g.Root.MxCell = nil
g.Root.MxCell = cells_without_links
}
func (cell *MxCell) processLinks() {
v := ""
if cell.Source == nil {
cell.Source = &v
}
if cell.Target == nil {
cell.Target = &v
}
}
type mxissue struct {
msg string
}

2
models/rtype/rtype.go
View File

@ -1,7 +1,7 @@
package rtype
import (
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
"cloud.o-forge.io/core/oc-catalog/services"
"github.com/beego/beego/v2/core/logs"
"go.mongodb.org/mongo-driver/mongo"
)

4
models/schedule.go
View File

@ -4,7 +4,7 @@ import (
"errors"
"time"
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
"cloud.o-forge.io/core/oc-catalog/services"
"github.com/beego/beego/v2/core/logs"
"github.com/vk496/cron"
"go.mongodb.org/mongo-driver/bson/primitive"
@ -99,7 +99,7 @@ func CheckSchedule(cronString string, duration uint, cronFirstDate, cronLastDate
return errors.New("Bad cron message: " + err.Error())
}
dcModel := GetDatacenterFromAcronym(services.DC_NAME)
dcModel := GetDatacenterFromAcronym(services.DC_NAME)
if dcModel == nil {
return errors.New("The DC " + services.DC_NAME + " doesn't have any DC model with that acronym")
}

2
models/search.go
View File

@ -3,7 +3,7 @@ package models
import (
"strings"
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
"cloud.o-forge.io/core/oc-catalog/services"
"github.com/beego/beego/v2/core/logs"
"go.mongodb.org/mongo-driver/bson"

4
models/storage.go
View File

@ -4,8 +4,8 @@ import (
"fmt"
"strings"
"cloud.o-forge.io/core/deprecated-oc-catalog/models/rtype"
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
"cloud.o-forge.io/core/oc-catalog/models/rtype"
"cloud.o-forge.io/core/oc-catalog/services"
"github.com/beego/beego/v2/core/logs"
"go.mongodb.org/mongo-driver/bson/primitive"
)

359
models/workflow.go
View File

@ -9,9 +9,9 @@ import (
"sort"
"time"
"cloud.o-forge.io/core/deprecated-oc-catalog/models/rtype"
swagger "cloud.o-forge.io/core/deprecated-oc-catalog/selfapi"
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
"cloud.o-forge.io/core/oc-catalog/models/rtype"
swagger "cloud.o-forge.io/core/oc-catalog/selfapi"
"cloud.o-forge.io/core/oc-catalog/services"
"github.com/beego/beego/v2/core/logs"
"github.com/vk496/cron"
"go.mongodb.org/mongo-driver/bson/primitive"
@ -56,15 +56,12 @@ type Workflow struct {
Computing map[string]ComputingObject `json:"computing"`
Storage map[string]StorageObject `json:"storage"`
Datacenter map[string]DatacenterObject `json:"datacenter"` //TODO: Decide if there should be multiple objects of a datacenter
Links map[string]Link `json:"link"`
Schedules WorkflowSchedule `json:"schedules"`
MxgraphXML string `description:"State of the mxgraph"`
}
// TODO : describe what use case this interface satisfies
type ResourceObject interface {
getHost() *string
getName() *string
@ -78,7 +75,7 @@ type ResourceObject interface {
// This type allows to process computing and storage component
// which can get input from the user
type EditableResourceObject interface {
type EditableResourceObject interface{
ResourceObject
addUserInput(map[string]interface{})
}
@ -119,13 +116,13 @@ func (w *Workflow) GetResource(rObjID *string) (retObj ResourceObject) {
return nil
}
if datVal, ok := w.Data[*rObjID]; ok {
retObj = &datVal
if storVal, ok := w.Data[*rObjID]; ok {
retObj = &storVal
return
}
if compVal, ok := w.Computing[*rObjID]; ok {
retObj = &compVal
if storVal, ok := w.Computing[*rObjID]; ok {
retObj = &storVal
return
}
@ -134,8 +131,8 @@ func (w *Workflow) GetResource(rObjID *string) (retObj ResourceObject) {
return
}
if dcVal, ok := w.Datacenter[*rObjID]; ok {
retObj = &dcVal
if storVal, ok := w.Datacenter[*rObjID]; ok {
retObj = &storVal
return
}
@ -182,13 +179,6 @@ func (w *Workflow) AddObj(robj ResourceObject) *primitive.ObjectID {
return &outputID
}
func (w *Workflow) AddLinkToWorkflow(link Link, id string) {
if w.Links == nil {
w.Links = make(map[string]Link)
}
w.Links[id] = link
}
func (w *Workflow) UpdateDB(userID, workflowName string) error {
_, err := services.MngoCollWorkspace.UpdateOne(services.MngoCtx,
@ -546,7 +536,9 @@ func ParseMxGraph(username, workflowName, xmlData string) (err error, mxissues [
return err, nil
}
xmlModel.createLinks()
// Move the attribute of the object's tags into the mxCell's for an easier processing
// currentWorkflow.extractMxCell(xmlModel)
targetWorkspaceWorkflow, err, mxissues := userWorkspace.ConsumeMxGraphModel(xmlModel)
if err != nil {
@ -594,6 +586,7 @@ func FindSliceInSlice(slice1 []string, slice2 []string) (int, int, bool) {
return -1, -1, false
}
// TODO : correct this method to suppport mxcells with settings
func (ws Workspace) ConsumeMxGraphModel(xmlmodel MxGraphModel) (returned_wf *Workflow, err error, issues []error) {
returned_wf = &Workflow{}
@ -604,38 +597,49 @@ func (ws Workspace) ConsumeMxGraphModel(xmlmodel MxGraphModel) (returned_wf *Wor
return xmlmodel.Root.MxCell[i].RID != nil
})
// Create the object and add it to the appropriate list
// for all the components with setting, which are identified
// by a MxObject tag in the xml
if ok := xmlmodel.Root.MxObject != nil; ok {
for _, object := range *xmlmodel.Root.MxObject {
resObj, err, mxissues := returned_wf.mxCellToComponent(object.MxCell, ws)
if err != nil {
issues = append(issues, mxissues...)
}
// add the component to the worflow's attribute that stores
// all components in a map[string]Component where the key
// is the component's ID in the mxGraph and the value the Component object
returned_wf.UpdateObj(resObj, object.ID)
// Construct the object corresponding to the componant's type and use its addUserInput method
if resObj.getRtype() == rtype.COMPUTING {
comp_obj := returned_wf.GetResource(&object.ID).(*ComputingObject)
comp_obj.AddUserInput(object.Settings)
returned_wf.UpdateObj(comp_obj, object.ID)
}
// if(resObj.getRtype() == rtype.DATA){
// }
}
}
// For each cell of the xml graph,
// in the case cell has a rID retrieve its rType from the value of rID of the component in the worfklow
// retrieve the component's type
// create an object from the rType
// update the existing workflow with the new component
// or by defautlt : the cell represents an arrow
// if the source or the target of the arrow is a datacenter
// define which end of the arrow is the DC
// if the other other end of the arrow is a computing component
// create a computing object
// attach the DC to it
// update the workflow with the object : create the list of this type of component or update the list with the id of the component with the object
for _, object := range *xmlmodel.Root.MxObject{
resObj, err, mxissues := returned_wf.mxCellToComponent(object.MxCell,ws)
if err != nil {
issues = append(issues, mxissues...)
}
// add the component to the worflow's attribute that stores
// all components in a map[string]Component where the key
// is the component's ID in the mxGraph and the value the Component object
returned_wf.UpdateObj(resObj,object.ID)
// Construct the object corresponding to the componant's type and use its addUserInput method
if(resObj.getRtype() == rtype.COMPUTING){
comp_obj := returned_wf.GetResource(&object.ID).(*ComputingObject)
comp_obj.AddUserInput(object.Settings)
returned_wf.UpdateObj(comp_obj,object.ID)
}
// if(resObj.getRtype() == rtype.DATA){
// }
}
for _, cell := range xmlmodel.Root.MxCell {
switch {
case cell.RID != nil:
resObj, err, mxissues := returned_wf.mxCellToComponent(cell, ws)
resObj, err, mxissues := returned_wf.mxCellToComponent(cell,ws)
if err != nil {
issues = append(issues, mxissues...)
}
@ -665,72 +669,93 @@ func (ws Workspace) ConsumeMxGraphModel(xmlmodel MxGraphModel) (returned_wf *Wor
continue
}
// Not root nor resource. Should be only links
// If is a invalid link, we can't save it in the DB
// We should always get a ID because we already registered resources and discarded which doesn't correspond to existent models
// save back
// If we have a relationship of:
// Source ----> Target
//
// The Source will be in the INPUTs of the Target.
// But we also must make sure that the Target will be in the OUTPUTs of the Source
if sourceObj.getRtype() == rtype.DATACENTER || targetObj.getRtype() == rtype.DATACENTER {
var datacenter, datacenterLinked *string
if sourceObj.getRtype() == rtype.DATACENTER {
datacenter = cell.Source
datacenterLinked = cell.Target
} else {
datacenter = cell.Target
datacenterLinked = cell.Source
}
switch returned_wf.GetResource(datacenterLinked).getRtype() {
case rtype.COMPUTING:
computingObj := returned_wf.GetResource(datacenterLinked).(*ComputingObject)
// We should always get a ID because we already registered resources and discarded which doesn't correspond to existent models
computingObj.DataCenterID = *datacenter
returned_wf.UpdateObj(computingObj, *datacenterLinked)
}
} else {
targetObj.addLink(INPUT, *cell.Source)
returned_wf.UpdateObj(targetObj, *cell.Target) // save back
// If we have a relationship of:
// Source ----> Target
//
// The Source will be in the INPUTs of the Target.
// But we also must make sure that the Target will be in the OUTPUTs of the Source
sourceObj.addLink(OUTPUT, *cell.Target)
returned_wf.UpdateObj(sourceObj, *cell.Source)
}
}
}
issues = returned_wf.createLinks(xmlmodel.Root.MxLink, issues)
issues = returned_wf.checkLinks(issues)
// dcslist := make(map[string]bool)
// dataslist := make(map[string]bool)
// // datalist := make(map[string]bool)
dcslist := make(map[string]bool)
dataslist := make(map[string]bool)
// datalist := make(map[string]bool)
// // Test wether the computing components are linked with a DC
// for _, comp := range returned_wf.Computing {
// if comp.DataCenterID == "" {
// issues = append(issues, errors.New("Computing "+*comp.getName()+" without a Datacenter"))
// } else {
// // If doesn't exist in the list, means is new element to register as used
// dcslist[comp.DataCenterID] = true
// Test wether the computing components are linked with a DC
for _, comp := range returned_wf.Computing {
if comp.DataCenterID == "" {
issues = append(issues, errors.New("Computing "+*comp.getName()+" without a Datacenter"))
} else {
// If doesn't exist in the list, means is new element to register as used
dcslist[comp.DataCenterID] = true
// }
}
// for _, dcin := range comp.Inputs {
// switch returned_wf.GetResource(&dcin).getRtype() {
// case rtype.DATA:
// dataslist[dcin] = true
// }
// }
for _, dcin := range comp.Inputs {
switch returned_wf.GetResource(&dcin).getRtype() {
case rtype.DATA:
dataslist[dcin] = true
}
}
// for _, dcout := range comp.Outputs {
// switch returned_wf.GetResource(&dcout).getRtype() {
// case rtype.DATA:
// dataslist[dcout] = true
// }
// }
for _, dcout := range comp.Outputs {
switch returned_wf.GetResource(&dcout).getRtype() {
case rtype.DATA:
dataslist[dcout] = true
}
}
// }
}
// for _, storage_component := range returned_wf.Storage {
// if storage_component.Inputs == nil && storage_component.Outputs == nil {
// issues = append(issues, errors.New("Storage "+*storage_component.getName()+" without compatible inputs and outputs"))
// }
// }
for _, storage_component := range returned_wf.Storage {
if storage_component.Inputs == nil && storage_component.Outputs == nil {
issues = append(issues, errors.New("Storage "+*storage_component.getName()+" without compatible inputs and outputs"))
}
}
// for dcID, dc_component := range returned_wf.Datacenter {
// // if rID doesn't exist in the list, it means that it's not used
// if _, ok := dcslist[dcID]; !ok {
// issues = append(issues, errors.New("DC "+*dc_component.getName()+" not attached to any Computing"))
// }
// }
for dcID, dc_component := range returned_wf.Datacenter {
// if rID doesn't exist in the list, it means that it's not used
if _, ok := dcslist[dcID]; !ok {
issues = append(issues, errors.New("DC "+*dc_component.getName()+" not atached to any Computing"))
}
}
// for dcID, data_component := range returned_wf.Data {
// // if rID doesn't exist in the list, it means that it's not used
// if _, ok := dataslist[dcID]; !ok {
// issues = append(issues, errors.New("Data "+*data_component.getName()+" not attached to any Computing"))
// }
// }
for dcID, data_component := range returned_wf.Data {
// if rID doesn't exist in the list, it means that it's not used
if _, ok := dataslist[dcID]; !ok {
issues = append(issues, errors.New("Data "+*data_component.getName()+" not atached to any Computing"))
}
}
//////////////////////////////////////////////////////////
// //
@ -833,115 +858,6 @@ func (ws Workspace) ConsumeMxGraphModel(xmlmodel MxGraphModel) (returned_wf *Wor
return
}
func (w *Workflow) createLinks(links []MxLink, issues []error) []error {
for _, link := range links {
if (len(link.Source) > 0 && len(link.Target) > 0){
sourceObj := w.GetResource(&link.Source)
targetObj := w.GetResource(&link.Target)
link_object := NewLink(sourceObj, link.Source, targetObj, link.Target)
w.AddLinkToWorkflow(link_object, link.ID)
} else {
issues = append(issues, w.processLinkErrors(link))
}
}
return issues
}
func (w *Workflow) processLinkErrors(link MxLink) (issue error) {
if len(link.Source) == 0 && len(link.Target) == 0 {
issue = errors.New("Arrow " + link.ID + " is alone")
} else if len(link.Source) == 0 {
targetObj := w.GetResource(&link.Target)
issue = errors.New("Arrow (" + link.ID + ") to " + *targetObj.getName() + " without parent")
} else {
sourceObj := w.GetResource(&link.Source)
issue = errors.New("Arrow " + link.ID + " from " + *sourceObj.getName() + " without target")
}
return issue
}
func (w *Workflow) checkLinks(issues []error) []error {
// Check that storage components have a valid link
for id, storage := range w.Storage {
if(!w.isComponentSrc(id) && !w.isComponentDst(id)){
issues = append(issues, errors.New("Storage "+*storage.getName()+" without compatible inputs and outputs"))
}
}
// Check that data components are linked to a computing component
for id, data := range w.Data {
if(!w.isLinkedToComputing(id)){
issues = append(issues, errors.New("Data "+*data.getName()+" not attached to any Computing"))
}
}
// Check that DC is linked to a computing component
for id, dc:= range w.Datacenter {
if(!w.isLinkedToComputing(id)){
issues = append(issues, errors.New("Datacenter "+*dc.getName()+" not attached to any Computing"))
}
}
// Check that all data computing components are linked to a DC
for id,comp:= range w.Computing {
if(!w.isLinkedToDC(id)){
issues = append(issues, errors.New("Computing "+*comp.getName()+" not attached to any datacenter"))
}
}
return issues
}
func (w *Workflow) isComponentSrc(id string) bool {
for _, link := range w.Links{
if(link.Source == id && link.Source != ""){
return true
}
}
return false
}
func (w *Workflow) isComponentDst(id string) bool {
for _, link := range w.Links{
if(link.Destination == id && link.Source != ""){
return true
}
}
return false
}
func (w *Workflow) isLinkedToComputing(id string) bool {
for idComputing, _ := range w.Computing {
if( (w.isComponentSrc(id) && w.isComponentDst(idComputing)) || (w.isComponentSrc(idComputing) && w.isComponentDst(id))){
return true
}
}
return false
}
func (w *Workflow) isLinkedToDC(id string) bool {
for _, link := range w.Links {
if link.Source == id && link.DCLink {
return true
}
}
return false
}
func sumExecutionReqs(exeqReq ...ExecutionRequirementsModel) (ret ExecutionRequirementsModel) {
for _, v := range exeqReq {
ret.CPUs += v.CPUs
@ -1123,15 +1039,15 @@ func CheckAndBookWorkflowSchedule(username, workflowName string, book bool) (myR
SchedulesDB: &currentWorkflow.Schedules}},
)
if err != nil {
if err != nil {
logs.Critical("Internal error when updating in DB: " + err.Error())
}
return myRet, nil
}
// Not sure if this method handles error propagation well
func (wf Workflow) mxCellToComponent(cell MxCell, ws Workspace) (resObj ResourceObject, err error, issues []error) {
// Not sure if this method handles error propagation well
func (wf Workflow) mxCellToComponent(cell MxCell, ws Workspace) (resObj ResourceObject,err error, issues []error){
rType := ws.getRtype(*cell.RID)
if rType == rtype.INVALID {
@ -1150,7 +1066,22 @@ func (wf Workflow) mxCellToComponent(cell MxCell, ws Workspace) (resObj Resource
resObj = wf.CreateResourceObject(rType)
resObj.setReference(rIDObj)
return
}
// func (ws Workspace) extractMxCell(xmlModel MxGraphModel){
// // Iterate through all objects of the MxGraph
// graphObjects := xmlModel.Root.MxObject
// for _, object := range(*graphObjects){
// current_obj_id, _ := strconv.Atoi(object.ID)
// inside_cell_id := strconv.Itoa(current_obj_id + 1)
// cell := ws.GetResource(&inside_cell_id)
// // component := w.GetResource(cell.RID)
// fmt.Print(cell)
// }
// // Extract the mxCell object
// // Invoke the addParameter method from the component
// // Edit the ID to get the object's one
// }

4
models/workspace.go
View File

@ -4,8 +4,8 @@ import (
"context"
"errors"
"cloud.o-forge.io/core/deprecated-oc-catalog/models/rtype"
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
"cloud.o-forge.io/core/oc-catalog/models/rtype"
"cloud.o-forge.io/core/oc-catalog/services"
"github.com/beego/beego/v2/core/logs"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"

4
routers/auth.go
View File

@ -4,8 +4,8 @@ import (
"reflect"
"strings"
"cloud.o-forge.io/core/deprecated-oc-catalog/controllers"
"cloud.o-forge.io/core/deprecated-oc-catalog/models"
"cloud.o-forge.io/core/oc-catalog/controllers"
"cloud.o-forge.io/core/oc-catalog/models"
"github.com/beego/beego/v2/core/logs"
beego "github.com/beego/beego/v2/server/web"
"github.com/beego/beego/v2/server/web/context"

4
routers/router.go
View File

@ -12,8 +12,8 @@ import (
"os"
"strings"
"cloud.o-forge.io/core/deprecated-oc-catalog/controllers"
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
"cloud.o-forge.io/core/oc-catalog/controllers"
"cloud.o-forge.io/core/oc-catalog/services"
"github.com/beego/beego/v2/core/logs"

37
scripts/demo.json
View File

@ -236,27 +236,6 @@
},
"inputs": [],
"outputs": []
},
{
"name": "Mosquito server",
"short_description": "open source message broker that implements the MQTT protocol versions 5.0, 3.1.1 and 3.1.",
"logo": "./local_imgs/mosquitto-logo.png",
"description": "A very long description of what this storage is",
"type": "computing",
"owner": "IRT",
"price": 300,
"license": "GPLv2",
"execution_requirements": {
"cpus": 1,
"ram": 1024,
"storage": 300,
"gpus": 1,
"disk_io": "30 MB/s",
"parallel": true,
"scaling_model": 2
},
"inputs": [],
"outputs": []
}
]
},
@ -294,6 +273,22 @@
"inputs": [],
"outputs": [],
"URL" : ""
},
{
"name": "Mosquito server",
"short_description": "open source message broker that implements the MQTT protocol versions 5.0, 3.1.1 and 3.1.",
"logo": "./local_imgs/mosquitto-logo.png",
"description": "A very long description of what this storage is",
"type": "storage",
"DCacronym": "DC_myDC",
"size": 40000,
"encryption": false,
"redundancy": "RAID5S",
"throughput": "r:300,w:350",
"bookingPrice": 90,
"inputs": [],
"outputs": [],
"URL" : ""
}
]
},

BIN
scripts/local_imgs/nginx.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 14 KiB

8
services/discovery.go
View File

@ -3,9 +3,7 @@ package services
import (
"os"
conf "cloud.o-forge.io/core/deprecated-oc-catalog/conf"
SelfAPI "cloud.o-forge.io/core/deprecated-oc-catalog/selfapi"
SelfAPI "cloud.o-forge.io/core/oc-catalog/selfapi"
"github.com/beego/beego/v2/core/logs"
beego "github.com/beego/beego/v2/server/web"
)
@ -16,7 +14,6 @@ func GetSelfAPI(host string) *SelfAPI.APIClient {
return SelfAPI.NewAPIClient(&SelfAPI.Configuration{BasePath: "http://" + host + "/v1"})
}
// Need to understand why there are so many different DC name
func Discoveryinit() {
dcNameOS := os.Getenv("DOCKER_DCNAME")
@ -35,7 +32,4 @@ func Discoveryinit() {
DC_NAME = "DC_DEFAULT"
logs.Warning("Default DC name is used")
// Quickfix for scheduling
DC_NAME = conf.GetConfig().DCNAME
}

2
services/init.go
View File

@ -1,7 +1,7 @@
package services
import (
"cloud.o-forge.io/core/deprecated-oc-catalog/conf"
"cloud.o-forge.io/core/oc-catalog/conf"
)
func Init() {

2
services/mongo.go
View File

@ -4,7 +4,7 @@ import (
"context"
"time"
"cloud.o-forge.io/core/deprecated-oc-catalog/conf"
"cloud.o-forge.io/core/oc-catalog/conf"
"github.com/beego/beego/v2/core/logs"
"go.mongodb.org/mongo-driver/mongo"