Compare commits

..

39 Commits

Author SHA1 Message Date
pb
c413e0f494 Téléverser les fichiers vers "scripts/local_imgs" 2024-08-26 15:44:44 +02:00
pb
c4efdc8bd9 updated import name with 'deprecated-' 2024-07-29 18:02:29 +02:00
pb
e4b5b8a341 Updated package name declaration
In mod.go changed the first line to module cloud.o-forge.io/core/deprecated-oc-catalog
2024-07-29 17:52:36 +02:00
pb
4b03211502 Merge remote-tracking branch 'refs/remotes/origin/pierre_configuration' 2024-07-04 16:08:05 +02:00
pb
af691638da Merge branch 'feature/sort_links' 2024-07-04 16:04:19 +02:00
pb
b87b6c9eeb fixing scheduling process 2024-07-04 15:57:28 +02:00
pb
b40879d8aa documentation 2024-07-04 15:56:36 +02:00
pb
a1f04dc2f0 bugfix empty workflow 2024-04-25 15:48:32 +02:00
pb
9bd5be7758 Merge branch 'argo_workflow' 2024-04-16 18:54:05 +02:00
pb
97c09f4eef remove dead code 2024-04-16 18:49:49 +02:00
pb
31c9c0c89d structtomap import 2024-04-16 16:16:15 +02:00
pb
d91afc2acd New class link to handle links between components and possible errors 2024-04-16 16:12:54 +02:00
pb
99278ff7ef minor indentation change 2024-04-15 11:43:07 +02:00
pb
64ae7eeb4c added links models to workflow 2024-04-15 11:42:17 +02:00
pb
ff9021b1ff differentiate links and cells post unmarshalling 2024-04-12 13:25:01 +02:00
pb
a07c7f2898 minor changes 2024-04-11 17:06:22 +02:00
pb
cba7b39216 minor changes 2024-04-11 17:06:22 +02:00
pb
590a5070e9 improved UML documentation 2024-04-11 17:06:22 +02:00
pb
57438ed215 removed wrong logic from previous change 2024-04-11 17:06:22 +02:00
pb
b51c0f1b74 update the object stored for one workflow with user input 2024-04-11 17:06:22 +02:00
pb
49d3ba9763 Retrieve component from mxCell inside object tag 2024-04-11 17:06:22 +02:00
pb
83c3c3d3f2 fixed go.mod 2024-04-11 17:06:22 +02:00
pb
1d2638f855 decomment some code to allow build 2024-04-11 17:06:22 +02:00
pb
baada840bf decomment some code to allow build 2024-04-11 17:06:22 +02:00
pb
c2c1c4827f remove thumbs.db:encryptable 2024-04-11 17:06:22 +02:00
pb
8c5e75855e new model for links 2024-04-11 16:34:43 +02:00
pb
9cd8af282f added some doc 2024-04-09 15:40:03 +02:00
pb
dbde933b74 fixed a typo 2024-04-09 15:39:32 +02:00
pb
20a2636677 Fixed router not working 2024-02-19 17:24:14 +01:00
pb
58d5493584 improved Dockerisation 2024-02-16 17:42:22 +01:00
pb
bd699aa0c9 added conf repertory to image 2024-02-15 18:04:55 +01:00
pb
4fcf84ce48 light dockerfile 2024-02-15 11:56:25 +01:00
pb
a1d3ddbbd0 corrected ports redirections in compose.backend 2024-02-15 11:49:44 +01:00
pb
7025c168ab Swagger returning only 404s 2024-02-15 11:03:32 +01:00
pb
a8d4e4bcf6 Modified DCName and DBPoint retrieving 2023-12-04 14:46:36 +01:00
pb
fdf1f20105 Solved the problem of generating swagger and minimal image (scratch) 2023-11-02 15:49:33 +01:00
pb
17f0998ffb Modified the conf loading process, now with onion in conf/conf.go 2023-10-31 11:23:39 +01:00
pb
1806624e76 Changed the conf object to the conf package for better accessibility 2023-10-31 10:30:01 +01:00
pb
31c7c44e2c started implementing a /ping route 2023-10-26 17:32:59 +02:00
39 changed files with 550 additions and 213 deletions

View File

@ -13,8 +13,8 @@ RUN bee generate routers
# Generating the swagger
RUN timeout 20 bee run -gendoc=true -downdoc=true -runmode=dev || :
RUN sed -i 's/http:\/\/127.0.0.1:8080\/swagger\/swagger.json/swagger.json/g' swagger/index.html &&\
sed -i 's/https:\/\/petstore.swagger.io\/v2\/swagger.json/swagger.json/g' swagger/index.html
RUN sed -i 's/http:\/\/127.0.0.1:8080\/swagger\/swagger.json/swagger.json/g' swagger/index.html
RUN sed -i 's/https:\/\/petstore.swagger.io\/v2\/swagger.json/swagger.json/g' swagger/index.html
RUN ls -l routers

View File

@ -31,6 +31,30 @@ const ui = SwaggerUIBundle({
This issue is fixed in the Dockerfile build with two `sed` to change the value of the swagger file URL.
## Deployment with Docker compose
## Running the app locally
In order to run the application we need to correct some errors that beego generates.
**Router generation**
Beego generates the routers from the comment of the controllers methods. However, there seems to be some errors when we let beego generate th routers from `bee run`.
We need to use `bee generate routers` to have a working router file.
**Swagger generation**
Using `bee run -downdoc=true -gendoc=true` beego download a swagger template and generate the JSON file (swagger.json) in `swagger/`. However the swagger template in `swagger/index.html` does not change the exemple's url for our current JSON file. When running locally we have to edit the call to construct the object holding the swagger information, with the parameter url set with the relative path to our JSON swagger file :
```
const ui = SwaggerUIBundle({
url: "swagger.json",
...
...
});
```
This issue is fixed in the Dockerfile build with two `sed` to change the value of the swagger file URL.
## Deployment with Docker compose
Deploy with docker:
@ -49,15 +73,18 @@ or out of docker `./scripts/populate_models.sh ./scripts/demo.json`
### Multinode
Deploy :
### Multinode
Deploy :
`docker-compose -f docker-compose.yml -f docker-compose.backend.yml -f docker-compose.multi.yml up --build`
## Populating the MongoDB database
From the root of the projet run :
On the machine running the mongoDB container, from the root of the project run this line :
`./scripts/multinode.sh ./scripts/demo.json`
`./scripts/populate_models.sh ./scripts/demo.json`
This script should be updated to be ran from anywhere.

View File

@ -1,7 +1,7 @@
package controllers
import (
"cloud.o-forge.io/core/oc-catalog/models"
"cloud.o-forge.io/core/deprecated-oc-catalog/models"
beego "github.com/beego/beego/v2/server/web"
)

View File

@ -1,7 +1,7 @@
package controllers
import (
"cloud.o-forge.io/core/oc-catalog/models"
"cloud.o-forge.io/core/deprecated-oc-catalog/models"
beego "github.com/beego/beego/v2/server/web"
"github.com/go-playground/validator/v10"

View File

@ -1,7 +1,7 @@
package controllers
import (
"cloud.o-forge.io/core/oc-catalog/models"
"cloud.o-forge.io/core/deprecated-oc-catalog/models"
beego "github.com/beego/beego/v2/server/web"
"github.com/go-playground/validator/v10"

View File

@ -3,7 +3,7 @@ package controllers
import (
"time"
"cloud.o-forge.io/core/oc-catalog/models"
"cloud.o-forge.io/core/deprecated-oc-catalog/models"
"github.com/beego/beego/v2/core/logs"
beego "github.com/beego/beego/v2/server/web"
)

View File

@ -1,7 +1,7 @@
package controllers
import (
"cloud.o-forge.io/core/oc-catalog/models"
"cloud.o-forge.io/core/deprecated-oc-catalog/models"
beego "github.com/beego/beego/v2/server/web"
)

View File

@ -1,7 +1,7 @@
package controllers
import (
"cloud.o-forge.io/core/oc-catalog/models"
"cloud.o-forge.io/core/deprecated-oc-catalog/models"
beego "github.com/beego/beego/v2/server/web"
)

View File

@ -1,7 +1,7 @@
package controllers
import (
"cloud.o-forge.io/core/oc-catalog/models"
"cloud.o-forge.io/core/deprecated-oc-catalog/models"
beego "github.com/beego/beego/v2/server/web"
)

View File

@ -3,7 +3,7 @@ package controllers
import (
"time"
"cloud.o-forge.io/core/oc-catalog/models"
"cloud.o-forge.io/core/deprecated-oc-catalog/models"
"github.com/beego/beego/v2/core/logs"
beego "github.com/beego/beego/v2/server/web"
"github.com/vk496/cron"

View File

@ -1,7 +1,7 @@
package controllers
import (
"cloud.o-forge.io/core/oc-catalog/models"
"cloud.o-forge.io/core/deprecated-oc-catalog/models"
beego "github.com/beego/beego/v2/server/web"
)

View File

@ -0,0 +1,47 @@
@startuml class_links
class LinkGraph {
+ []Link links
+ void AddLinkToGraph(Link link)
+ boolean,str HasPrevious(Link)
+ boolean,str HasNext(Link)
}
class Link {
+ str source
+ str destination
+ boolean DCLink
+ *Link NewLink(interface{} src, interface{} dst)
+ void AddLinkToDataCenter()
}
note left of LinkGraph::HasPrevious
checks if the component whose ID is in src is the dst of
any other Link of the list on Link
end note
note top of Link
Links need to be redefined in the sense that they are currently used
both :
- to connect a component to a DC
- to represent the interractions between components
end note
note right of Link::DCLink
set at construction of the object and used to order the links
end note
note left of Link::NewLink
Must test if the parameters check the type constraints
and raise errors for the GUI if necessary
end note
LinkGraph*--"links"Link
@enduml

54
docs/UML/mxgraph.puml Normal file
View File

@ -0,0 +1,54 @@
@startuml mxgraph
package MxGraph {
class MxGraphModel {
- XMLName xml.Name
+ Root : RootStruct
+ createLinks()
}
class RootStruct {
- XMLName xml.Name
- MxCell[] MxCell
- MxObject[] MxObject
- MxLink[] MxLink
}
class MxCell {
- XMLName xml.Name
+ ID string
+ RID *string
+ Rtype string
+ Parent *string
+ Edge *string
+ Source *string
+ Target *string
+ processLinks()
}
class MxLink {
+ ID string
+ Source string
+ Target string
}
class MxObject {
- XMLName xml.Name
+ ID string
+ Settings []xml.Attr
+ MxCell MxCell
}
class mxissue {
- msg string
+ Error() string
}
MxGraphModel *-- RootStruct : Root
RootStruct *-- MxCell : MxCell
RootStruct *-- MxObject : MxObject
RootStruct *-- MxLink : MxLink
MxObject *-- MxCell : MxCell
}
@enduml

View File

@ -0,0 +1,2 @@
@startuml
@enduml

View File

@ -0,0 +1,5 @@
For an easier parsing and constrution of dependencies we decided that :
- Computing elements must be linked between one another, to represent the chaining of events
- A storage component can be shared by several computing component, the direction of the arrow will determine if it's for writting or reading

View File

@ -2,11 +2,12 @@
- [ ] In most of the components from 'models/' we have a method to add input and output to the model, however this linking of components is already done in oc-schedule when parsing the MxGraph. We need to determine if adding relations between components inside the objects themself is necessary.
- When running in debug mode with a breakpoint inside the first line of computing.addLink it is only called once
- [ ]
## MxGraph
- [ ] The ConsumeMxGraphModel is way too long, it should refactored and broken down in different sub methods
- mxcell are put inside an <object> tag when the settings have been opened, wether values have been set or not. Maybe we could find a way to make mxgraph add these whenever we add a component to the graph.
- then identify the links only
- [ ] It is unclear what are the inputs and the ouputs. It seems like they were implemented to link two components, but it seems redundant with the identification of links
- [ ] It is unclear what are the inputs and the ouputs. It seems like they were implemented to link two components, but it seems redundant with the identification of links
- This has been potentially tackled with the creation of a class to handle links between components. The components do no handle their own connections with other components, this task is delegated to the Link and Worlflow classes.

25
docs/linking_errors.md Normal file
View File

@ -0,0 +1,25 @@
# Handling errors during workflows' post
Every time an user modify a worflow through oc-search's GUI its representation is sent through an XML to oc-catalog API.
To ensure a correct execution of the workflow we must look for irregularities in the workflow's design. Of course an error can be the result of a workflow being currently built by the user, with the corrective action coming.
This document aims at laying down all the requirements that each component must respect and show wether they have been implemented in the code or not.
## Computing
- [x] A computing component must be paired with a datacenter component
## Data
- [x] A data component must be linked to at least one computing component
## Datacenter
- [x] A datacenter component must be linked to at least one computing component
## Storage
- [x] A storage component must have at least one target or be the source of another component

3
go.mod
View File

@ -1,10 +1,11 @@
module cloud.o-forge.io/core/oc-catalog
module cloud.o-forge.io/core/deprecated-oc-catalog
go 1.15
require github.com/beego/beego/v2 v2.0.1
require (
github.com/Klathmon/StructToMap v0.0.0-20140724123129-3d0229e2dce7
github.com/antihax/optional v1.0.0
github.com/aws/aws-sdk-go v1.36.29 // indirect
github.com/beego/beego v1.12.12

2
go.sum
View File

@ -1,6 +1,8 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Klathmon/StructToMap v0.0.0-20140724123129-3d0229e2dce7 h1:n0MD6UkwbgGHtXsmfgVzC2+ZbHzIsScpbq9ZGI18074=
github.com/Klathmon/StructToMap v0.0.0-20140724123129-3d0229e2dce7/go.mod h1:xdrQDwHlKUmv8yiElMx6W0W10cLkqpeSEUUib8KGtv4=
github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=

View File

@ -3,9 +3,9 @@ package main
import (
"os"
"cloud.o-forge.io/core/oc-catalog/conf"
"cloud.o-forge.io/core/oc-catalog/routers"
"cloud.o-forge.io/core/oc-catalog/services"
"cloud.o-forge.io/core/deprecated-oc-catalog/conf"
"cloud.o-forge.io/core/deprecated-oc-catalog/routers"
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
"github.com/beego/beego/logs"
beego "github.com/beego/beego/v2/server/web"

View File

@ -5,8 +5,8 @@ import (
"fmt"
"strings"
"cloud.o-forge.io/core/oc-catalog/models/rtype"
"cloud.o-forge.io/core/oc-catalog/services"
"cloud.o-forge.io/core/deprecated-oc-catalog/models/rtype"
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
structtomap "github.com/Klathmon/StructToMap"
"github.com/beego/beego/v2/core/logs"
"go.mongodb.org/mongo-driver/bson/primitive"
@ -77,15 +77,15 @@ func (model ComputingModel) getName() string {
type ComputingObject struct {
ReferenceID primitive.ObjectID `json:"referenceID" description:"Computing model ID"`
Inputs []string `json:"inputs"`
Outputs []string `json:"outputs"`
Inputs []string `json:"inputs"`
Outputs []string `json:"outputs"`
Image string `json:"image,omitempty"`
Command string `json:"command,omitempty"`
Arguments []string `json:"arguments,omitempty"`
Environment []string `json:"environment,omitempty"`
Ports []string `json:"ports,omitempty"`
DataCenterID string `json:"datacenterID" description:"Datacenter where the computing will be executed"`
DataCenterID string `json:"datacenterID" description:"Datacenter where the computing will be executed"`
}
func (obj ComputingObject) getHost() *string {

View File

@ -1,8 +1,8 @@
package models
import (
"cloud.o-forge.io/core/oc-catalog/models/rtype"
"cloud.o-forge.io/core/oc-catalog/services"
"cloud.o-forge.io/core/deprecated-oc-catalog/models/rtype"
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
"github.com/beego/beego/v2/core/logs"
"go.mongodb.org/mongo-driver/bson/primitive"
)

View File

@ -4,8 +4,8 @@ import (
"net"
"time"
"cloud.o-forge.io/core/oc-catalog/models/rtype"
"cloud.o-forge.io/core/oc-catalog/services"
"cloud.o-forge.io/core/deprecated-oc-catalog/models/rtype"
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
"github.com/beego/beego/v2/core/logs"
"go.mongodb.org/mongo-driver/bson/primitive"
)

View File

@ -3,8 +3,8 @@ package models
import (
"errors"
"cloud.o-forge.io/core/oc-catalog/models/rtype"
"cloud.o-forge.io/core/oc-catalog/services"
"cloud.o-forge.io/core/deprecated-oc-catalog/models/rtype"
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
"github.com/beego/beego/v2/core/logs"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"

53
models/links.go Normal file
View File

@ -0,0 +1,53 @@
package models
import (
"cloud.o-forge.io/core/deprecated-oc-catalog/models/rtype"
)
type Link struct {
// ID primitive.ObjectID `json:"ID" bson:"_id" required:"true" example:"5099803df3f4948bd2f98391"`
Source string `json:"source" description:"id in the workflow of the source object"`
Destination string `json:"destination" description:"id in the workflow of the destination object"`
DCLink bool `json:"dcLink" description:"is this a link with a datacenter"`
}
// Use ResourceObject parameter to process certain components type differently
// and Id's to identify each component as a node in an oriented graph
// In the case of DCLink we choose to always consider the DC as the destination
// in order to facilitate some logic
func NewLink(src ResourceObject, srcId string, dst ResourceObject, dstId string) (link Link) {
link.Source = srcId
link.Destination = dstId
// If the link is between a DC and a component make sure that the DC is destination
// and if the component is computing, update the DataCenterID
if (src.getRtype() == rtype.DATACENTER || dst.getRtype() == rtype.DATACENTER){
var linked ResourceObject
link.DCLink = true
if src.getRtype() == rtype.DATACENTER {
linked = dst
} else {
linked = src
}
if( link.DCLink && src.getRtype() == rtype.DATACENTER){
link.Destination = srcId
link.Source = dstId
}
if (linked.getRtype() == rtype.COMPUTING){
linked.(*ComputingObject).DataCenterID = link.Destination
}
}
return
}

View File

@ -8,22 +8,29 @@ type MxGraphModel struct {
XMLName xml.Name `xml:"mxGraphModel"`
Root struct {
XMLName xml.Name `xml:"root"`
MxCell []MxCell `xml:"mxCell"`
XMLName xml.Name `xml:"root"`
MxCell []MxCell `xml:"mxCell"`
MxObject *[]MxObject `xml:"object"`
MxLink []MxLink
}
}
type MxCell struct {
XMLName xml.Name `xml:"mxCell"`
ID string `xml:"id,attr"`
Parent *string `xml:"parent,attr"`
RID *string `xml:"rID,attr"`
Source *string `xml:"source,attr"`
Target *string `xml:"target,attr"`
Rtype *string `xml:"rType,attr"`
XMLName xml.Name `xml:"mxCell"`
ID string `xml:"id,attr"`
RID *string `xml:"rID,attr"`
Rtype string `xml:"rType,attr"`
Parent *string `xml:"parent,attr"`
Edge *string `xml:"edge,attr"`
Source *string `xml:"source,attr"`
Target *string `xml:"target,attr"`
}
type MxLink struct {
ID string `xml:"id,attr"`
Source string `xml:"source,attr"`
Target string `xml:"target,attr"`
}
type MxObject struct {
XMLName xml.Name `xml:"object"`
@ -32,6 +39,39 @@ type MxObject struct {
MxCell MxCell `xml:"mxCell"`
}
// Didn't manage to differentiate Links and cells containing components using
// only structures and unmarshal, so we use this method post-umarshalling
func (g *MxGraphModel) createLinks() {
var cells_without_links []MxCell
for i, mxcell := range g.Root.MxCell {
if mxcell.Edge != nil {
mxcell.processLinks()
newLink := MxLink{mxcell.ID,*mxcell.Source,*mxcell.Target}
g.Root.MxLink = append(g.Root.MxLink,newLink)
} else {
cells_without_links = append(cells_without_links,g.Root.MxCell[i])
}
}
g.Root.MxCell = nil
g.Root.MxCell = cells_without_links
}
func (cell *MxCell) processLinks() {
v := ""
if cell.Source == nil {
cell.Source = &v
}
if cell.Target == nil {
cell.Target = &v
}
}
type mxissue struct {
msg string
}

View File

@ -1,7 +1,7 @@
package rtype
import (
"cloud.o-forge.io/core/oc-catalog/services"
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
"github.com/beego/beego/v2/core/logs"
"go.mongodb.org/mongo-driver/mongo"
)

View File

@ -4,7 +4,7 @@ import (
"errors"
"time"
"cloud.o-forge.io/core/oc-catalog/services"
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
"github.com/beego/beego/v2/core/logs"
"github.com/vk496/cron"
"go.mongodb.org/mongo-driver/bson/primitive"
@ -99,7 +99,7 @@ func CheckSchedule(cronString string, duration uint, cronFirstDate, cronLastDate
return errors.New("Bad cron message: " + err.Error())
}
dcModel := GetDatacenterFromAcronym(services.DC_NAME)
dcModel := GetDatacenterFromAcronym(services.DC_NAME)
if dcModel == nil {
return errors.New("The DC " + services.DC_NAME + " doesn't have any DC model with that acronym")
}

View File

@ -3,7 +3,7 @@ package models
import (
"strings"
"cloud.o-forge.io/core/oc-catalog/services"
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
"github.com/beego/beego/v2/core/logs"
"go.mongodb.org/mongo-driver/bson"

View File

@ -4,8 +4,8 @@ import (
"fmt"
"strings"
"cloud.o-forge.io/core/oc-catalog/models/rtype"
"cloud.o-forge.io/core/oc-catalog/services"
"cloud.o-forge.io/core/deprecated-oc-catalog/models/rtype"
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
"github.com/beego/beego/v2/core/logs"
"go.mongodb.org/mongo-driver/bson/primitive"
)

View File

@ -9,9 +9,9 @@ import (
"sort"
"time"
"cloud.o-forge.io/core/oc-catalog/models/rtype"
swagger "cloud.o-forge.io/core/oc-catalog/selfapi"
"cloud.o-forge.io/core/oc-catalog/services"
"cloud.o-forge.io/core/deprecated-oc-catalog/models/rtype"
swagger "cloud.o-forge.io/core/deprecated-oc-catalog/selfapi"
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
"github.com/beego/beego/v2/core/logs"
"github.com/vk496/cron"
"go.mongodb.org/mongo-driver/bson/primitive"
@ -56,12 +56,15 @@ type Workflow struct {
Computing map[string]ComputingObject `json:"computing"`
Storage map[string]StorageObject `json:"storage"`
Datacenter map[string]DatacenterObject `json:"datacenter"` //TODO: Decide if there should be multiple objects of a datacenter
Links map[string]Link `json:"link"`
Schedules WorkflowSchedule `json:"schedules"`
MxgraphXML string `description:"State of the mxgraph"`
}
// TODO : describe what use case this interface satisfies
type ResourceObject interface {
getHost() *string
getName() *string
@ -75,7 +78,7 @@ type ResourceObject interface {
// This type allows to process computing and storage component
// which can get input from the user
type EditableResourceObject interface{
type EditableResourceObject interface {
ResourceObject
addUserInput(map[string]interface{})
}
@ -116,13 +119,13 @@ func (w *Workflow) GetResource(rObjID *string) (retObj ResourceObject) {
return nil
}
if storVal, ok := w.Data[*rObjID]; ok {
retObj = &storVal
if datVal, ok := w.Data[*rObjID]; ok {
retObj = &datVal
return
}
if storVal, ok := w.Computing[*rObjID]; ok {
retObj = &storVal
if compVal, ok := w.Computing[*rObjID]; ok {
retObj = &compVal
return
}
@ -131,8 +134,8 @@ func (w *Workflow) GetResource(rObjID *string) (retObj ResourceObject) {
return
}
if storVal, ok := w.Datacenter[*rObjID]; ok {
retObj = &storVal
if dcVal, ok := w.Datacenter[*rObjID]; ok {
retObj = &dcVal
return
}
@ -179,6 +182,13 @@ func (w *Workflow) AddObj(robj ResourceObject) *primitive.ObjectID {
return &outputID
}
func (w *Workflow) AddLinkToWorkflow(link Link, id string) {
if w.Links == nil {
w.Links = make(map[string]Link)
}
w.Links[id] = link
}
func (w *Workflow) UpdateDB(userID, workflowName string) error {
_, err := services.MngoCollWorkspace.UpdateOne(services.MngoCtx,
@ -536,9 +546,7 @@ func ParseMxGraph(username, workflowName, xmlData string) (err error, mxissues [
return err, nil
}
// Move the attribute of the object's tags into the mxCell's for an easier processing
// currentWorkflow.extractMxCell(xmlModel)
xmlModel.createLinks()
targetWorkspaceWorkflow, err, mxissues := userWorkspace.ConsumeMxGraphModel(xmlModel)
if err != nil {
@ -586,7 +594,6 @@ func FindSliceInSlice(slice1 []string, slice2 []string) (int, int, bool) {
return -1, -1, false
}
// TODO : correct this method to suppport mxcells with settings
func (ws Workspace) ConsumeMxGraphModel(xmlmodel MxGraphModel) (returned_wf *Workflow, err error, issues []error) {
returned_wf = &Workflow{}
@ -597,49 +604,38 @@ func (ws Workspace) ConsumeMxGraphModel(xmlmodel MxGraphModel) (returned_wf *Wor
return xmlmodel.Root.MxCell[i].RID != nil
})
// For each cell of the xml graph,
// in the case cell has a rID retrieve its rType from the value of rID of the component in the worfklow
// retrieve the component's type
// create an object from the rType
// update the existing workflow with the new component
// or by defautlt : the cell represents an arrow
// if the source or the target of the arrow is a datacenter
// define which end of the arrow is the DC
// if the other other end of the arrow is a computing component
// create a computing object
// attach the DC to it
// update the workflow with the object : create the list of this type of component or update the list with the id of the component with the object
// Create the object and add it to the appropriate list
// for all the components with setting, which are identified
// by a MxObject tag in the xml
if ok := xmlmodel.Root.MxObject != nil; ok {
for _, object := range *xmlmodel.Root.MxObject {
for _, object := range *xmlmodel.Root.MxObject{
resObj, err, mxissues := returned_wf.mxCellToComponent(object.MxCell,ws)
if err != nil {
issues = append(issues, mxissues...)
resObj, err, mxissues := returned_wf.mxCellToComponent(object.MxCell, ws)
if err != nil {
issues = append(issues, mxissues...)
}
// add the component to the worflow's attribute that stores
// all components in a map[string]Component where the key
// is the component's ID in the mxGraph and the value the Component object
returned_wf.UpdateObj(resObj, object.ID)
// Construct the object corresponding to the componant's type and use its addUserInput method
if resObj.getRtype() == rtype.COMPUTING {
comp_obj := returned_wf.GetResource(&object.ID).(*ComputingObject)
comp_obj.AddUserInput(object.Settings)
returned_wf.UpdateObj(comp_obj, object.ID)
}
// if(resObj.getRtype() == rtype.DATA){
// }
}
// add the component to the worflow's attribute that stores
// all components in a map[string]Component where the key
// is the component's ID in the mxGraph and the value the Component object
returned_wf.UpdateObj(resObj,object.ID)
// Construct the object corresponding to the componant's type and use its addUserInput method
if(resObj.getRtype() == rtype.COMPUTING){
comp_obj := returned_wf.GetResource(&object.ID).(*ComputingObject)
comp_obj.AddUserInput(object.Settings)
returned_wf.UpdateObj(comp_obj,object.ID)
}
// if(resObj.getRtype() == rtype.DATA){
// }
}
for _, cell := range xmlmodel.Root.MxCell {
switch {
case cell.RID != nil:
resObj, err, mxissues := returned_wf.mxCellToComponent(cell,ws)
resObj, err, mxissues := returned_wf.mxCellToComponent(cell, ws)
if err != nil {
issues = append(issues, mxissues...)
}
@ -669,93 +665,72 @@ func (ws Workspace) ConsumeMxGraphModel(xmlmodel MxGraphModel) (returned_wf *Wor
continue
}
if sourceObj.getRtype() == rtype.DATACENTER || targetObj.getRtype() == rtype.DATACENTER {
var datacenter, datacenterLinked *string
if sourceObj.getRtype() == rtype.DATACENTER {
datacenter = cell.Source
datacenterLinked = cell.Target
} else {
datacenter = cell.Target
datacenterLinked = cell.Source
}
switch returned_wf.GetResource(datacenterLinked).getRtype() {
case rtype.COMPUTING:
computingObj := returned_wf.GetResource(datacenterLinked).(*ComputingObject)
// We should always get a ID because we already registered resources and discarded which doesn't correspond to existent models
computingObj.DataCenterID = *datacenter
returned_wf.UpdateObj(computingObj, *datacenterLinked)
}
} else {
targetObj.addLink(INPUT, *cell.Source)
returned_wf.UpdateObj(targetObj, *cell.Target) // save back
// If we have a relationship of:
// Source ----> Target
//
// The Source will be in the INPUTs of the Target.
// But we also must make sure that the Target will be in the OUTPUTs of the Source
sourceObj.addLink(OUTPUT, *cell.Target)
returned_wf.UpdateObj(sourceObj, *cell.Source)
}
// Not root nor resource. Should be only links
// If is a invalid link, we can't save it in the DB
// We should always get a ID because we already registered resources and discarded which doesn't correspond to existent models
// save back
// If we have a relationship of:
// Source ----> Target
//
// The Source will be in the INPUTs of the Target.
// But we also must make sure that the Target will be in the OUTPUTs of the Source
}
}
dcslist := make(map[string]bool)
dataslist := make(map[string]bool)
// datalist := make(map[string]bool)
issues = returned_wf.createLinks(xmlmodel.Root.MxLink, issues)
issues = returned_wf.checkLinks(issues)
// dcslist := make(map[string]bool)
// dataslist := make(map[string]bool)
// // datalist := make(map[string]bool)
// Test wether the computing components are linked with a DC
for _, comp := range returned_wf.Computing {
if comp.DataCenterID == "" {
issues = append(issues, errors.New("Computing "+*comp.getName()+" without a Datacenter"))
} else {
// If doesn't exist in the list, means is new element to register as used
dcslist[comp.DataCenterID] = true
// // Test wether the computing components are linked with a DC
// for _, comp := range returned_wf.Computing {
// if comp.DataCenterID == "" {
// issues = append(issues, errors.New("Computing "+*comp.getName()+" without a Datacenter"))
// } else {
// // If doesn't exist in the list, means is new element to register as used
// dcslist[comp.DataCenterID] = true
}
// }
for _, dcin := range comp.Inputs {
switch returned_wf.GetResource(&dcin).getRtype() {
case rtype.DATA:
dataslist[dcin] = true
}
}
// for _, dcin := range comp.Inputs {
// switch returned_wf.GetResource(&dcin).getRtype() {
// case rtype.DATA:
// dataslist[dcin] = true
// }
// }
for _, dcout := range comp.Outputs {
switch returned_wf.GetResource(&dcout).getRtype() {
case rtype.DATA:
dataslist[dcout] = true
}
}
// for _, dcout := range comp.Outputs {
// switch returned_wf.GetResource(&dcout).getRtype() {
// case rtype.DATA:
// dataslist[dcout] = true
// }
// }
}
// }
for _, storage_component := range returned_wf.Storage {
if storage_component.Inputs == nil && storage_component.Outputs == nil {
issues = append(issues, errors.New("Storage "+*storage_component.getName()+" without compatible inputs and outputs"))
}
}
// for _, storage_component := range returned_wf.Storage {
// if storage_component.Inputs == nil && storage_component.Outputs == nil {
// issues = append(issues, errors.New("Storage "+*storage_component.getName()+" without compatible inputs and outputs"))
// }
// }
for dcID, dc_component := range returned_wf.Datacenter {
// if rID doesn't exist in the list, it means that it's not used
if _, ok := dcslist[dcID]; !ok {
issues = append(issues, errors.New("DC "+*dc_component.getName()+" not atached to any Computing"))
}
}
// for dcID, dc_component := range returned_wf.Datacenter {
// // if rID doesn't exist in the list, it means that it's not used
// if _, ok := dcslist[dcID]; !ok {
// issues = append(issues, errors.New("DC "+*dc_component.getName()+" not attached to any Computing"))
// }
// }
for dcID, data_component := range returned_wf.Data {
// if rID doesn't exist in the list, it means that it's not used
if _, ok := dataslist[dcID]; !ok {
issues = append(issues, errors.New("Data "+*data_component.getName()+" not atached to any Computing"))
}
}
// for dcID, data_component := range returned_wf.Data {
// // if rID doesn't exist in the list, it means that it's not used
// if _, ok := dataslist[dcID]; !ok {
// issues = append(issues, errors.New("Data "+*data_component.getName()+" not attached to any Computing"))
// }
// }
//////////////////////////////////////////////////////////
// //
@ -858,6 +833,115 @@ func (ws Workspace) ConsumeMxGraphModel(xmlmodel MxGraphModel) (returned_wf *Wor
return
}
func (w *Workflow) createLinks(links []MxLink, issues []error) []error {
for _, link := range links {
if (len(link.Source) > 0 && len(link.Target) > 0){
sourceObj := w.GetResource(&link.Source)
targetObj := w.GetResource(&link.Target)
link_object := NewLink(sourceObj, link.Source, targetObj, link.Target)
w.AddLinkToWorkflow(link_object, link.ID)
} else {
issues = append(issues, w.processLinkErrors(link))
}
}
return issues
}
func (w *Workflow) processLinkErrors(link MxLink) (issue error) {
if len(link.Source) == 0 && len(link.Target) == 0 {
issue = errors.New("Arrow " + link.ID + " is alone")
} else if len(link.Source) == 0 {
targetObj := w.GetResource(&link.Target)
issue = errors.New("Arrow (" + link.ID + ") to " + *targetObj.getName() + " without parent")
} else {
sourceObj := w.GetResource(&link.Source)
issue = errors.New("Arrow " + link.ID + " from " + *sourceObj.getName() + " without target")
}
return issue
}
func (w *Workflow) checkLinks(issues []error) []error {
// Check that storage components have a valid link
for id, storage := range w.Storage {
if(!w.isComponentSrc(id) && !w.isComponentDst(id)){
issues = append(issues, errors.New("Storage "+*storage.getName()+" without compatible inputs and outputs"))
}
}
// Check that data components are linked to a computing component
for id, data := range w.Data {
if(!w.isLinkedToComputing(id)){
issues = append(issues, errors.New("Data "+*data.getName()+" not attached to any Computing"))
}
}
// Check that DC is linked to a computing component
for id, dc:= range w.Datacenter {
if(!w.isLinkedToComputing(id)){
issues = append(issues, errors.New("Datacenter "+*dc.getName()+" not attached to any Computing"))
}
}
// Check that all data computing components are linked to a DC
for id,comp:= range w.Computing {
if(!w.isLinkedToDC(id)){
issues = append(issues, errors.New("Computing "+*comp.getName()+" not attached to any datacenter"))
}
}
return issues
}
func (w *Workflow) isComponentSrc(id string) bool {
for _, link := range w.Links{
if(link.Source == id && link.Source != ""){
return true
}
}
return false
}
func (w *Workflow) isComponentDst(id string) bool {
for _, link := range w.Links{
if(link.Destination == id && link.Source != ""){
return true
}
}
return false
}
func (w *Workflow) isLinkedToComputing(id string) bool {
for idComputing, _ := range w.Computing {
if( (w.isComponentSrc(id) && w.isComponentDst(idComputing)) || (w.isComponentSrc(idComputing) && w.isComponentDst(id))){
return true
}
}
return false
}
func (w *Workflow) isLinkedToDC(id string) bool {
for _, link := range w.Links {
if link.Source == id && link.DCLink {
return true
}
}
return false
}
func sumExecutionReqs(exeqReq ...ExecutionRequirementsModel) (ret ExecutionRequirementsModel) {
for _, v := range exeqReq {
ret.CPUs += v.CPUs
@ -1039,15 +1123,15 @@ func CheckAndBookWorkflowSchedule(username, workflowName string, book bool) (myR
SchedulesDB: &currentWorkflow.Schedules}},
)
if err != nil {
if err != nil {
logs.Critical("Internal error when updating in DB: " + err.Error())
}
return myRet, nil
}
// Not sure if this method handles error propagation well
func (wf Workflow) mxCellToComponent(cell MxCell, ws Workspace) (resObj ResourceObject,err error, issues []error){
// Not sure if this method handles error propagation well
func (wf Workflow) mxCellToComponent(cell MxCell, ws Workspace) (resObj ResourceObject, err error, issues []error) {
rType := ws.getRtype(*cell.RID)
if rType == rtype.INVALID {
@ -1066,22 +1150,7 @@ func (wf Workflow) mxCellToComponent(cell MxCell, ws Workspace) (resObj Resource
resObj = wf.CreateResourceObject(rType)
resObj.setReference(rIDObj)
return
}
// func (ws Workspace) extractMxCell(xmlModel MxGraphModel){
// // Iterate through all objects of the MxGraph
// graphObjects := xmlModel.Root.MxObject
// for _, object := range(*graphObjects){
// current_obj_id, _ := strconv.Atoi(object.ID)
// inside_cell_id := strconv.Itoa(current_obj_id + 1)
// cell := ws.GetResource(&inside_cell_id)
// // component := w.GetResource(cell.RID)
// fmt.Print(cell)
// }
// // Extract the mxCell object
// // Invoke the addParameter method from the component
// // Edit the ID to get the object's one
// }

View File

@ -4,8 +4,8 @@ import (
"context"
"errors"
"cloud.o-forge.io/core/oc-catalog/models/rtype"
"cloud.o-forge.io/core/oc-catalog/services"
"cloud.o-forge.io/core/deprecated-oc-catalog/models/rtype"
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
"github.com/beego/beego/v2/core/logs"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"

View File

@ -4,8 +4,8 @@ import (
"reflect"
"strings"
"cloud.o-forge.io/core/oc-catalog/controllers"
"cloud.o-forge.io/core/oc-catalog/models"
"cloud.o-forge.io/core/deprecated-oc-catalog/controllers"
"cloud.o-forge.io/core/deprecated-oc-catalog/models"
"github.com/beego/beego/v2/core/logs"
beego "github.com/beego/beego/v2/server/web"
"github.com/beego/beego/v2/server/web/context"

View File

@ -12,8 +12,8 @@ import (
"os"
"strings"
"cloud.o-forge.io/core/oc-catalog/controllers"
"cloud.o-forge.io/core/oc-catalog/services"
"cloud.o-forge.io/core/deprecated-oc-catalog/controllers"
"cloud.o-forge.io/core/deprecated-oc-catalog/services"
"github.com/beego/beego/v2/core/logs"

View File

@ -236,6 +236,27 @@
},
"inputs": [],
"outputs": []
},
{
"name": "Mosquito server",
"short_description": "open source message broker that implements the MQTT protocol versions 5.0, 3.1.1 and 3.1.",
"logo": "./local_imgs/mosquitto-logo.png",
"description": "A very long description of what this storage is",
"type": "computing",
"owner": "IRT",
"price": 300,
"license": "GPLv2",
"execution_requirements": {
"cpus": 1,
"ram": 1024,
"storage": 300,
"gpus": 1,
"disk_io": "30 MB/s",
"parallel": true,
"scaling_model": 2
},
"inputs": [],
"outputs": []
}
]
},
@ -273,22 +294,6 @@
"inputs": [],
"outputs": [],
"URL" : ""
},
{
"name": "Mosquito server",
"short_description": "open source message broker that implements the MQTT protocol versions 5.0, 3.1.1 and 3.1.",
"logo": "./local_imgs/mosquitto-logo.png",
"description": "A very long description of what this storage is",
"type": "storage",
"DCacronym": "DC_myDC",
"size": 40000,
"encryption": false,
"redundancy": "RAID5S",
"throughput": "r:300,w:350",
"bookingPrice": 90,
"inputs": [],
"outputs": [],
"URL" : ""
}
]
},

Binary file not shown.

After

Width:  |  Height:  |  Size: 14 KiB

View File

@ -3,7 +3,9 @@ package services
import (
"os"
SelfAPI "cloud.o-forge.io/core/oc-catalog/selfapi"
conf "cloud.o-forge.io/core/deprecated-oc-catalog/conf"
SelfAPI "cloud.o-forge.io/core/deprecated-oc-catalog/selfapi"
"github.com/beego/beego/v2/core/logs"
beego "github.com/beego/beego/v2/server/web"
)
@ -14,6 +16,7 @@ func GetSelfAPI(host string) *SelfAPI.APIClient {
return SelfAPI.NewAPIClient(&SelfAPI.Configuration{BasePath: "http://" + host + "/v1"})
}
// Need to understand why there are so many different DC name
func Discoveryinit() {
dcNameOS := os.Getenv("DOCKER_DCNAME")
@ -32,4 +35,7 @@ func Discoveryinit() {
DC_NAME = "DC_DEFAULT"
logs.Warning("Default DC name is used")
// Quickfix for scheduling
DC_NAME = conf.GetConfig().DCNAME
}

View File

@ -1,7 +1,7 @@
package services
import (
"cloud.o-forge.io/core/oc-catalog/conf"
"cloud.o-forge.io/core/deprecated-oc-catalog/conf"
)
func Init() {

View File

@ -4,7 +4,7 @@ import (
"context"
"time"
"cloud.o-forge.io/core/oc-catalog/conf"
"cloud.o-forge.io/core/deprecated-oc-catalog/conf"
"github.com/beego/beego/v2/core/logs"
"go.mongodb.org/mongo-driver/mongo"