2024-07-23 12:16:20 +02:00
package workflow_builder
2023-10-18 17:08:53 +02:00
import (
"encoding/json"
"fmt"
2024-04-30 14:43:38 +02:00
"maps"
2023-10-18 17:08:53 +02:00
2024-07-23 12:16:20 +02:00
"oc-scheduler/conf"
"oc-scheduler/logger"
models "oc-scheduler/models"
catalog_models "cloud.o-forge.io/core/oc-catalog/models" // this will be replaced with oc-lib
2024-04-09 11:20:08 +02:00
"github.com/beego/beego/v2/core/logs"
2023-10-18 17:08:53 +02:00
"github.com/tidwall/gjson"
)
2024-07-02 12:24:23 +02:00
2023-10-18 17:08:53 +02:00
type Graph struct {
2024-07-23 12:16:20 +02:00
workflow_name string // used to test if the graph has been instatiated, private so can only be set by a graph's method
Datas [ ] catalog_models . DataModel
Computings [ ] catalog_models . ComputingModel
Datacenters [ ] catalog_models . DatacenterModel
Storages [ ] catalog_models . StorageModel
Links map [ string ] catalog_models . Link
ws models . HttpQuery
2023-10-18 17:08:53 +02:00
}
2024-07-03 10:22:22 +02:00
// Create a dictionnaries with each existing workflow from a workspace, associated to the JSON representation of its content
2023-10-18 17:08:53 +02:00
func ( g * Graph ) GetGraphList ( apiurl string ) ( map [ string ] string , error ) {
g . ws . Init ( apiurl )
body , err := g . ws . Get ( "v1/workspace/list" )
if err != nil {
return nil , err
}
workspaces := make ( map [ string ] string )
result := gjson . Get ( string ( body ) , "Workflows" )
result . ForEach ( func ( key , value gjson . Result ) bool {
workspaces [ key . Str ] = value . String ( )
return true // keep iterating
} )
return workspaces , nil
}
2024-07-23 12:16:20 +02:00
// Should the parameter be removed, since we have oc-catalog url in the conf ?
func ( g * Graph ) GetGraph ( apiurl string , workflow string ) ( string , error ) {
g . ws . Init ( apiurl )
body , err := g . ws . Get ( "v1/workflow/" + workflow )
if err != nil {
return "" , err
}
graph := string ( body )
// result := gjson.Get(string(body), "Workflows")
// result.ForEach(func(key, value gjson.Result) bool {
// workspaces[key.Str] = value.String()
// return true // keep iterating
// })
return graph , nil
}
2024-04-04 12:31:12 +02:00
// Create the objects from the mxgraphxml stored in the workflow given as a parameter
2024-07-23 12:16:20 +02:00
func ( g * Graph ) LoadFrom ( workflow_name string ) error {
2024-04-04 12:31:12 +02:00
// Extract the xmlgraph from the given workspace
2024-07-23 12:16:20 +02:00
graph , err := g . GetGraph ( conf . GetConfig ( ) . OcCatalogUrl , workflow_name )
2023-10-18 17:08:53 +02:00
if err != nil {
return err
}
2024-07-03 10:22:22 +02:00
// os.WriteFile("graph.xml", []byte(decodedValue), 0660)
2024-04-04 12:31:12 +02:00
2024-07-23 12:16:20 +02:00
g . GetWorkflowComponents ( graph )
g . GetLinks ( graph )
2024-04-04 12:31:12 +02:00
2024-07-23 12:16:20 +02:00
g . workflow_name = workflow_name
2024-05-07 11:43:12 +02:00
2023-10-18 17:08:53 +02:00
return nil
}
2024-04-04 12:31:12 +02:00
2024-04-09 11:20:08 +02:00
// Create the objects that correspond to each component
// in a workflow, combining the user input and the base components attributes
func ( g * Graph ) GetWorkflowComponents ( workflow string ) {
types := [ ] string { "computing" , "datacenter" , "data" , "storage" } // create a constant for more maintainability OR even better get the list of all component's type for this WF
for _ , component_type := range types {
// Retrieve the dict of component for a specific type in the workflow
result := gjson . Get ( workflow , component_type )
if ( result . Type != gjson . Null ) {
result . ForEach ( func ( id , value gjson . Result ) bool {
2024-04-09 14:38:52 +02:00
comp_id := value . Get ( "referenceID" ) . Str
2024-04-09 11:20:08 +02:00
if ( comp_id != "" ) {
switch component_type {
case "computing" :
2024-04-09 15:03:12 +02:00
g . AddComputingModel ( comp_id , value , id . Str )
2024-04-09 11:20:08 +02:00
case "data" :
2024-04-09 15:03:12 +02:00
g . AddDataModel ( comp_id , value , id . Str )
2024-04-09 11:20:08 +02:00
case "datacenter" :
2024-04-09 15:03:12 +02:00
g . AddDatacenterModel ( comp_id , value , id . Str )
2024-04-09 11:20:08 +02:00
case "storage" :
2024-04-09 15:03:12 +02:00
g . AddStorageModel ( comp_id , value , id . Str )
2024-04-09 11:20:08 +02:00
default :
logs . Critical ( "Component type doesn't match a know type : " + component_type )
}
}
return true
} )
}
}
}
2024-04-04 12:31:12 +02:00
2024-04-16 19:38:21 +02:00
func ( g * Graph ) GetLinks ( workflow string ) {
2024-07-23 12:16:20 +02:00
g . Links = make ( map [ string ] catalog_models . Link )
2024-04-16 19:38:21 +02:00
result := gjson . Get ( workflow , "link" )
if ( result . Type != gjson . Null ) {
result . ForEach ( func ( id , value gjson . Result ) bool {
2024-07-23 12:16:20 +02:00
var l catalog_models . Link
2024-04-19 16:13:41 +02:00
2024-04-30 14:43:38 +02:00
json . Unmarshal ( [ ] byte ( value . Raw ) , & l )
2024-04-19 16:13:41 +02:00
g . Links [ id . Str ] = l
2024-04-16 19:38:21 +02:00
return true
} )
}
}
2024-04-09 15:03:12 +02:00
func ( g * Graph ) AddDataModel ( id string , user_input gjson . Result , wf_id string ) error {
2024-07-23 12:16:20 +02:00
var d catalog_models . DataModel
2023-10-18 17:08:53 +02:00
resp , err := g . ws . Get ( "v1/data/" + id )
if err != nil {
return err
}
json . Unmarshal ( resp , & d )
2024-04-09 14:38:52 +02:00
json . Unmarshal ( [ ] byte ( user_input . Raw ) , & d . DataNEWModel )
2024-04-09 15:03:12 +02:00
d . ID = wf_id
2023-10-18 17:08:53 +02:00
g . Datas = append ( g . Datas , d )
return nil
}
2024-04-09 15:03:12 +02:00
func ( g * Graph ) AddDatacenterModel ( id string , user_input gjson . Result , wf_id string ) error {
2024-07-23 12:16:20 +02:00
var d catalog_models . DatacenterModel
2023-10-18 17:08:53 +02:00
resp , err := g . ws . Get ( "v1/datacenter/" + id )
if err != nil {
return err
}
json . Unmarshal ( resp , & d )
2024-04-09 14:38:52 +02:00
json . Unmarshal ( [ ] byte ( user_input . Raw ) , & d . DatacenterNEWModel )
2024-04-09 15:03:12 +02:00
d . ID = wf_id
2023-10-18 17:08:53 +02:00
g . Datacenters = append ( g . Datacenters , d )
return nil
}
2024-04-09 15:03:12 +02:00
func ( g * Graph ) AddComputingModel ( id string , user_input gjson . Result , wf_id string ) error {
2024-07-23 12:16:20 +02:00
var c catalog_models . ComputingModel
2023-10-18 17:08:53 +02:00
resp , err := g . ws . Get ( "v1/computing/" + id )
if err != nil {
return err
}
json . Unmarshal ( resp , & c )
2024-04-09 14:38:52 +02:00
json . Unmarshal ( [ ] byte ( user_input . Raw ) , & c . ComputingNEWModel )
2024-04-09 15:03:12 +02:00
c . ID = wf_id
2023-10-18 17:08:53 +02:00
g . Computings = append ( g . Computings , c )
return nil
}
2024-04-09 15:03:12 +02:00
func ( g * Graph ) AddStorageModel ( id string , user_input gjson . Result , wf_id string ) error {
2024-07-23 12:16:20 +02:00
var s catalog_models . StorageModel
2024-04-30 14:43:38 +02:00
resp , err := g . ws . Get ( "v1/storage/" + id )
2023-10-18 17:08:53 +02:00
if err != nil {
return err
}
json . Unmarshal ( resp , & s )
2024-04-09 14:38:52 +02:00
json . Unmarshal ( [ ] byte ( user_input . Raw ) , & s . StorageNEWModel )
2024-04-09 15:03:12 +02:00
s . ID = wf_id
2023-10-18 17:08:53 +02:00
g . Storages = append ( g . Storages , s )
return nil
}
2024-07-23 12:16:20 +02:00
func ( g * Graph ) ExportToArgo ( ) ( string , error ) {
if len ( g . workflow_name ) == 0 {
return "" , fmt . Errorf ( "can't export a graph that has not been loaded yet" )
}
end_links := make ( map [ string ] catalog_models . Link )
2024-04-30 14:43:38 +02:00
2024-04-19 16:13:41 +02:00
for i , link := range g . Links {
if ( ! link . DCLink && ! g . isSource ( link . Destination , i ) ) {
end_links [ i ] = link
}
}
2024-04-30 14:43:38 +02:00
// index_list := make([]int, len(g.Links))
// list_branches := make([][]string,0)
list_branches := g . getListBranches ( end_links , nil , nil )
for _ , branch := range list_branches {
str := ""
for _ , link := range branch {
str = str + " --> " + g . getComponentName ( g . Links [ link ] . Source ) + " linked with " + g . getComponentName ( g . Links [ link ] . Destination )
}
fmt . Println ( str )
}
2024-05-03 10:58:06 +02:00
2024-04-30 14:43:38 +02:00
fmt . Println ( "Identified branches : " , list_branches )
2024-05-03 10:58:06 +02:00
argo_builder := ArgoBuilder { graph : * g , branches : list_branches }
2024-07-23 12:16:20 +02:00
filename , err := argo_builder . CreateDAG ( )
if err != nil {
logger . Logger . Error ( ) . Msg ( "Could not create the argo file for " + g . workflow_name )
return "" , err
}
return filename , nil
2024-04-30 14:43:38 +02:00
}
// Return a list containing the IDs of each link that make up a branch in the graph
2024-07-23 12:16:20 +02:00
func ( g * Graph ) getListBranches ( end_links map [ string ] catalog_models . Link , unvisited_links_list map [ string ] catalog_models . Link , current_branch [ ] string ) ( list_branches [ ] [ ] string ) {
2024-05-16 14:18:06 +02:00
2024-04-30 14:43:38 +02:00
if current_branch == nil {
current_branch = make ( [ ] string , 0 )
}
2024-05-03 10:58:06 +02:00
if unvisited_links_list == nil {
2024-07-23 12:16:20 +02:00
unvisited_links_list = make ( map [ string ] catalog_models . Link , len ( g . Links ) )
2024-05-03 10:58:06 +02:00
maps . Copy ( unvisited_links_list , g . Links )
fmt . Println ( unvisited_links_list )
2024-04-30 14:43:38 +02:00
}
for link_id , _ := range end_links {
j := link_id
new_branches := make ( [ ] [ ] string , 0 )
2024-04-19 16:13:41 +02:00
2024-05-03 10:58:06 +02:00
previous_index := g . getPreviousLink ( j , unvisited_links_list )
2024-04-30 14:43:38 +02:00
if len ( previous_index ) == 0 {
2024-05-02 09:52:28 +02:00
list_branches = append ( list_branches , [ ] string { link_id } )
2024-04-30 14:43:38 +02:00
}
for _ , id_link := range previous_index {
current_branch = append ( [ ] string { link_id } , current_branch ... )
2024-05-03 10:58:06 +02:00
delete ( unvisited_links_list , link_id )
2024-04-30 14:43:38 +02:00
// create a new branch for each previous link, appending the current path to this node to the created branch
2024-07-23 12:16:20 +02:00
new_end_link := make ( map [ string ] catalog_models . Link , 0 )
2024-04-30 14:43:38 +02:00
new_end_link [ id_link ] = g . Links [ id_link ]
2024-05-03 10:58:06 +02:00
new_branches = g . getListBranches ( new_end_link , unvisited_links_list , current_branch )
2024-04-30 14:43:38 +02:00
for _ , new_branch := range new_branches {
current_branch = append ( new_branch , link_id )
list_branches = append ( list_branches , current_branch )
2024-04-19 16:13:41 +02:00
}
}
}
2024-04-30 14:43:38 +02:00
return
2023-10-18 17:08:53 +02:00
}
func ( g * Graph ) ExportToHelm ( id string ) error {
return nil
}
2024-04-04 12:31:12 +02:00
2024-04-19 16:13:41 +02:00
// Return if it exists a link where Destination is the same as comp_id
func ( g * Graph ) isDestination ( comp_id string , link_id string ) bool {
for i , link := range g . Links {
if ( i != link_id && link . Destination == comp_id ) {
return true
}
}
return false
}
// Return if it exists a link where Source is the same as comp_id
func ( g * Graph ) isSource ( comp_id string , link_id string ) bool {
for i , link := range g . Links {
if ( i != link_id && link . Source == comp_id && ! link . DCLink ) {
return true
}
}
return false
}
// Returns an index number if their is a link in g.Links
// with the same Destination id that the Source id in g.Links[linkIndex]
// or nil if not
2024-07-23 12:16:20 +02:00
func ( g * Graph ) getPreviousLink ( link_id string , map_link map [ string ] catalog_models . Link ) ( previous_id [ ] string ) {
2024-04-30 14:43:38 +02:00
for k , link := range map_link {
if ( k != link_id && link . Destination == g . Links [ link_id ] . Source ) {
previous_id = append ( previous_id , k )
2024-04-19 16:13:41 +02:00
}
}
2024-04-30 14:43:38 +02:00
return
2024-04-19 16:13:41 +02:00
}
func ( g * Graph ) getComponentName ( id string ) string {
for _ , comp := range g . Computings {
if comp . ID == id {
return comp . Name
}
}
for _ , storage := range g . Storages {
if storage . ID == id {
return storage . Name
}
}
for _ , data := range g . Datas {
if data . ID == id {
return data . Name
}
}
return ""
}
2024-05-16 14:18:06 +02:00
// returns either computing, data or storage
2024-04-30 14:43:38 +02:00
func ( g * Graph ) getComponentType ( component_id string ) string {
for _ , comp := range g . Computings {
if comp . ID == component_id {
return "computing"
}
}
for _ , data := range g . Datas {
if data . ID == component_id {
return "data"
}
}
for _ , storage := range g . Storages {
if storage . ID == component_id {
return "storage"
}
}
return ""
}
// Returns a slice of id, in case the link is made of twice the same type of component
2024-07-23 12:16:20 +02:00
func ( g * Graph ) getComponentByType ( compType string , link catalog_models . Link ) ( ids [ ] string ) {
2024-04-30 14:43:38 +02:00
if ( g . getComponentType ( link . Source ) == compType ) {
ids = append ( ids , link . Source )
}
if ( g . getComponentType ( link . Destination ) == compType ) {
ids = append ( ids , link . Destination )
}
return
}