Changed component's retrieving from parsing XML to using the stored info in DB

This commit is contained in:
pb 2024-04-09 11:20:08 +02:00
parent 23ee20a59f
commit 86062f3d88
4 changed files with 226 additions and 88 deletions

View File

@ -1,2 +1,18 @@
# oc-scheduler
OC-Scheduler retrieves the content of submitted workflows and prepare them to be executed.
## Parsing
From a workflow's name we retrieve the xml graph associated and parse it in order to create the object representing each componant.
Each object is linked to another, represented by a links object with the two object IDs has attributes.
TODO :
- [x] Retrieve the user input's for each component.
## Organising
TODO :
- [ ] create an argo file from the graph/worfklow
- [ ] Create a different entry for each component
- [ ] execute each element in the right order

86
componentParser.go Normal file
View File

@ -0,0 +1,86 @@
package main
import (
"fmt"
"cloud.o-forge.io/core/oc-catalog/models"
"github.com/mitchellh/mapstructure"
)
// This module allows us to :
// - Reduce the size of graph.go
// - apply specific object construction logic depending on the component's type
// So far the unpacking of json/map data into the models has the caveat of not applying to the nested struct
func ConstructComputingObject(id string, component_info map[string]interface{}, user_inputs map[string]interface{}) (computing models.ComputingModel) {
computing.ID = id
err := mapstructure.Decode(component_info,&computing.ComputingNEWModel)
if err != nil {
fmt.Println("Computing : Error unpacking json into objects info")
log.Err(err)
}
err = mapstructure.Decode(user_inputs,&computing.ComputingNEWModel)
if err != nil {
fmt.Println("Computing : Error unpacking data input into comp object")
log.Err(err).Msg(err.Error())
}
return
}
func ConstructDataObject(id string, component_info map[string]interface{}, user_inputs map[string]interface{}) (data models.DataModel) {
data.ID = id
err := mapstructure.Decode(component_info,&data.DataNEWModel)
if err != nil {
fmt.Println("Data: Error unpacking json into objects info")
log.Err(err)
}
err = mapstructure.Decode(user_inputs,&data.DataNEWModel)
if err != nil {
fmt.Println("Data: Error unpacking data input into comp object")
log.Err(err).Msg(err.Error())
}
return
}
func ConstructDatacenterObject(id string, component_info map[string]interface{}, user_inputs map[string]interface{}) (datacenter models.DatacenterModel) {
datacenter.ID = id
err := mapstructure.Decode(component_info,&datacenter.DatacenterNEWModel)
if err != nil {
fmt.Println("Datacenter: Error unpacking json into objects info")
log.Err(err)
}
err = mapstructure.Decode(user_inputs,&datacenter.DatacenterNEWModel)
if err != nil {
fmt.Println("Datacenter: Error unpacking data input into comp object")
log.Err(err).Msg(err.Error())
}
return
}
func ConstructStorageObject(id string, component_info map[string]interface{}, user_inputs map[string]interface{}) (storage models.StorageModel) {
storage.ID = id
err := mapstructure.Decode(component_info,&storage.StorageNEWModel)
if err != nil {
fmt.Println("Storage: Error unpacking json into objects info")
log.Err(err)
}
err = mapstructure.Decode(user_inputs,&storage.StorageNEWModel)
if err != nil {
fmt.Println("Storage: Error unpacking data input into comp object")
log.Err(err).Msg(err.Error())
}
return
}

205
graph.go
View File

@ -8,7 +8,8 @@ import (
"strings"
"cloud.o-forge.io/core/oc-catalog/models"
"github.com/mitchellh/mapstructure"
"github.com/beego/beego/v2/core/logs"
"github.com/sbabiv/xml2map"
"github.com/tidwall/gjson"
)
@ -83,46 +84,47 @@ func (g *Graph) LoadFrom(workspace string) error {
g.GetWorkflowComponents(workspace)
for _, element := range cells {
id := element["@id"].(string)
// Case MXCell
if _, ok := element["@style"]; ok {
if _, ok2 := element["@rID"]; ok2 {
// Resolve elements
// fmt.Print(id + ": ")
// fmt.Println(element["@rID"], element["@rType"])
// fmt.Println(element)
dictionnary[id] = element["@rID"].(string)
g.addElementByType(element)
}
} else {
// Case object : contains user's input through the GUI
if _, ok := element["mxCell"]; ok {
// Attribute values
// for _, element := range cells {
// id := element["@id"].(string)
// // Case MXCell
// if _, ok := element["@style"]; ok {
// if _, ok2 := element["@rID"]; ok2 {
// // Resolve elements
// // fmt.Print(id + ": ")
// // fmt.Println(element["@rID"], element["@rType"])
// // fmt.Println(element)
// dictionnary[id] = element["@rID"].(string)
// g.addElementByType(element)
// }
// } else {
// // Case object : contains user's input through the GUI
// if _, ok := element["mxCell"]; ok {
// // Attribute values
// Extracts the cell ids
element = element["mxCell"].(map[string]interface{})
if _, ok := element["@style"]; ok {
if _, ok2 := element["@rID"]; ok2 {
// Resolve elements
// fmt.Print(id + ": ")
// fmt.Println(element["@rID"], element["@rType"])
// fmt.Println(element)
dictionnary[id] = element["@rID"].(string)
g.addElementByType(element)
}
}
}
// // Extracts the cell ids
// element = element["mxCell"].(map[string]interface{})
// if _, ok := element["@style"]; ok {
// if _, ok2 := element["@rID"]; ok2 {
// // Resolve elements
// // fmt.Print(id + ": ")
// // fmt.Println(element["@rID"], element["@rType"])
// // fmt.Println(element)
// dictionnary[id] = element["@rID"].(string)
// g.addElementByType(element)
// }
// }
// }
}
// register links
if src, ok := element["@source"]; ok {
//src = element["@source"].(string)
idlinks = append(idlinks, Link{Src: src.(string), Dst: element["@target"].(string)})
// fmt.Println("Link: " + src.(string) + " " + element["@target"].(string))
}
// }
// // register links
// if src, ok := element["@source"]; ok {
// //src = element["@source"].(string)
// idlinks = append(idlinks, Link{Src: src.(string), Dst: element["@target"].(string)})
// // fmt.Println("Link: " + src.(string) + " " + element["@target"].(string))
// }
// }
}
// translate links
for _, link := range idlinks {
g.Links = append(g.Links, Link{Src: dictionnary[link.Src], Dst: dictionnary[link.Dst]})
@ -140,48 +142,96 @@ func (g *Graph) addElementByType(element map[string]interface{}) {
}
if element["@rType"] == "computing" {
g.AddComputingModel(element["@rID"].(string))
g.AddUserInput(element["@rID"].(string))
}
if element["@rType"] == "storage" {
g.AddStorageModel(element["@rID"].(string))
}
}
func (g *Graph) GetWorkflowComponents(workspace string){
// retrieve if list of components' object
result := gjson.Get(workspace, "computing")
os.WriteFile("computing.json", []byte(result.String()), 0660)
// TODO : extract all the JSON/data processing to a new object that takes
// the current graph as an attribute and deals with adding new objects
// to it, depending on their type
result.ForEach(func(id, value gjson.Result) bool{
m, ok := value.Value().(map[string]interface{})
if !ok {
fmt.Printf("error getting map from json")
}
// Create the objects that correspond to each component
// in a workflow, combining the user input and the base components attributes
var comp_obj models.ComputingModel
comp_obj.ID = id.Str
err := mapstructure.Decode(m,&comp_obj.ComputingNEWModel)
if err != nil {
fmt.Print("Error unpacking json into comp object")
}
func (g *Graph) GetWorkflowComponents(workflow string){
types := []string{"computing","datacenter","data","storage"} // create a constant for more maintainability OR even better get the list of all component's type for this WF
for _, component_type := range types {
// Retrieve the dict of component for a specific type in the workflow
result := gjson.Get(workflow, component_type)
if (result.Type != gjson.Null) {
result.ForEach(func(id, value gjson.Result) bool{
component_db, ok := value.Value().(map[string]interface{})
if !ok {
fmt.Printf("error getting map from json")
}
g.Computings = append(g.Computings, comp_obj)
comp_id := component_db["referenceID"].(string)
var obj interface{}
if (comp_id != "") {
component_map, _ := g.GetComponentById(component_type, comp_id)
switch component_type {
case "computing":
obj = ConstructComputingObject(id.Str,component_map,component_db)
case "data":
obj = ConstructDataObject(id.Str,component_map,component_db)
case "datacenter":
obj = ConstructDatacenterObject(id.Str,component_map,component_db)
case "storage":
obj = ConstructStorageObject(id.Str,component_map,component_db)
default :
logs.Critical("Component type doesn't match a know type : " + component_type)
}
g.AddComponentObject(component_type, obj)
}
return true
})
}
}
}
return true
func (g *Graph) AddComponentObject(comp_type string, component interface{}){
switch comp_type {
case "computing":
g.Computings = append(g.Computings, component.(models.ComputingModel))
case "data":
g.Datas = append(g.Datas, component.(models.DataModel))
case "datacenter":
g.Datacenters = append(g.Datacenters, component.(models.DatacenterModel))
case "storage":
g.Storages = append(g.Storages, component.(models.StorageModel))
}
}
// Construct the object corresponding to component_type from its ID, retrieved in
// the xml graph, in order to merge the user input with the base model
func (g *Graph) GetComponentById(component_type string, id string) (map[string]interface{}, error) {
// TODO : Add a verification that g.ws is initialized ?
body , err := g.ws.Get("v1/"+component_type+"/"+id)
if err != nil {
return nil, err
}
comp := make(map[string]interface{})
jsonified := gjson.ParseBytes(body)
jsonified.ForEach(func(key, value gjson.Result) bool {
comp[key.Str] = value.String()
return true // keep iterating
})
// var slice_obj []models.ComputingModel
// var response map[string]interface{}
// json.Unmarshal(computings_obj,response)
// if len(computings_obj) != 0 {
// for _, obj := range(computings_obj){
// // var new_comp models.ComputingModel
// // json.Unmarshal(obj,&new_comp)
// fmt.Print(obj)
// }
// }
return comp, nil
}
func (g *Graph) AddDataModel(id string) error {
@ -228,25 +278,6 @@ func (g *Graph) AddStorageModel(id string) error {
return nil
}
// Add the user input to the object
// So far only computing (command, args, env var) and storage (path to the storing point) allow user input
func (g *Graph) AddUserInput(id string) error {
// TODO refactor the code in oc-catalog models so that we can call the addUserInput regardless of the class
// Detect which type of component
// Retrieve the component from its ID
fmt.Printf("component %s",id)
// switch type
// case computing
// Computing.addUserInput(map[string]interface{})
// case storage
// Storage..addUserInput(map[string]interface{})
return nil
}
func (g *Graph) ExportToArgo(id string) error {
return nil
}

View File

@ -1,6 +1,7 @@
package main
import (
"fmt"
"os"
"time"
@ -28,6 +29,7 @@ func main() {
_ = loglevel
var g Graph
list, err := g.GetGraphList(apiurl)
if err != nil {
log.Fatal().Msg("Failed to get the workspaces list, check api url and that api server is up : " + apiurl)
@ -35,6 +37,9 @@ func main() {
for workspace, _ := range list {
println(workspace)
}
g.LoadFrom(list["TLE-feed"])
g.LoadFrom(list["test-alpr"])
fmt.Print("stop")
}