implemented oclib and generates argo yaml
This commit is contained in:
parent
9ecea6dece
commit
aaac37a883
46
argo_workflows/chalitic-scobeele_01_08_2024_183304.yml
Normal file
46
argo_workflows/chalitic-scobeele_01_08_2024_183304.yml
Normal file
@ -0,0 +1,46 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Workflow
|
||||||
|
metadata:
|
||||||
|
generateName: oc-test-chalitic-scobeele
|
||||||
|
spec:
|
||||||
|
entrypoint: dag
|
||||||
|
volumeClaimTemplates:
|
||||||
|
- metadata:
|
||||||
|
name: workdir
|
||||||
|
spec:
|
||||||
|
accessModes: [ReadWriteOnce]
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 1Gi
|
||||||
|
templates:
|
||||||
|
- name: "-0"
|
||||||
|
container:
|
||||||
|
image: ""
|
||||||
|
args: [""]
|
||||||
|
volumeMounts:
|
||||||
|
- name: workdir
|
||||||
|
mountPath: /mnt/vol
|
||||||
|
- name: curl-1
|
||||||
|
container:
|
||||||
|
image: curlimages/curl:7.88.1
|
||||||
|
args: [-SL, 'https://toulousefc.com', -o, /mnt/vol/tfc.hmtl]
|
||||||
|
volumeMounts:
|
||||||
|
- name: workdir
|
||||||
|
mountPath: /mnt/vol
|
||||||
|
- name: alpine-2
|
||||||
|
container:
|
||||||
|
image: alpine:3.7
|
||||||
|
command: [sh, -c]
|
||||||
|
args: ['grep ''<meta name="description"'' /mnt/vol/tfc.html | awk -F ''content="'' ''{print $2}'' | awk -F ''"'' ''{print $1}''']
|
||||||
|
volumeMounts:
|
||||||
|
- name: workdir
|
||||||
|
mountPath: /mnt/vol
|
||||||
|
- name: dag
|
||||||
|
dag:
|
||||||
|
tasks:
|
||||||
|
- name: "-0"
|
||||||
|
template: "-0"
|
||||||
|
- name: curl-1
|
||||||
|
template: curl-1
|
||||||
|
- name: alpine-2
|
||||||
|
template: alpine-2
|
46
argo_workflows/colintic-schicke_01_08_2024_183508.yml
Normal file
46
argo_workflows/colintic-schicke_01_08_2024_183508.yml
Normal file
@ -0,0 +1,46 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Workflow
|
||||||
|
metadata:
|
||||||
|
generateName: oc-test-colintic-schicke
|
||||||
|
spec:
|
||||||
|
entrypoint: dag
|
||||||
|
volumeClaimTemplates:
|
||||||
|
- metadata:
|
||||||
|
name: workdir
|
||||||
|
spec:
|
||||||
|
accessModes: [ReadWriteOnce]
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 1Gi
|
||||||
|
templates:
|
||||||
|
- name: "-0"
|
||||||
|
container:
|
||||||
|
image: ""
|
||||||
|
args: [""]
|
||||||
|
volumeMounts:
|
||||||
|
- name: workdir
|
||||||
|
mountPath: /mnt/vol
|
||||||
|
- name: curl-1
|
||||||
|
container:
|
||||||
|
image: curlimages/curl:7.88.1
|
||||||
|
args: [-SL, 'https://toulousefc.com', -o, /mnt/vol/tfc.hmtl]
|
||||||
|
volumeMounts:
|
||||||
|
- name: workdir
|
||||||
|
mountPath: /mnt/vol
|
||||||
|
- name: alpine-2
|
||||||
|
container:
|
||||||
|
image: alpine:3.7
|
||||||
|
command: [sh, -c]
|
||||||
|
args: ['grep ''<meta name="description"'' /mnt/vol/tfc.html | awk -F ''content="'' ''{print $2}'' | awk -F ''"'' ''{print $1}''']
|
||||||
|
volumeMounts:
|
||||||
|
- name: workdir
|
||||||
|
mountPath: /mnt/vol
|
||||||
|
- name: dag
|
||||||
|
dag:
|
||||||
|
tasks:
|
||||||
|
- name: "-0"
|
||||||
|
template: "-0"
|
||||||
|
- name: curl-1
|
||||||
|
template: curl-1
|
||||||
|
- name: alpine-2
|
||||||
|
template: alpine-2
|
39
argo_workflows/sismon-oximous_02_08_2024_131457.yml
Normal file
39
argo_workflows/sismon-oximous_02_08_2024_131457.yml
Normal file
@ -0,0 +1,39 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Workflow
|
||||||
|
metadata:
|
||||||
|
generateName: oc-test-sismon-oximous
|
||||||
|
spec:
|
||||||
|
entrypoint: dag
|
||||||
|
volumeClaimTemplates:
|
||||||
|
- metadata:
|
||||||
|
name: workdir
|
||||||
|
spec:
|
||||||
|
accessModes: [ReadWriteOnce]
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 1Gi
|
||||||
|
templates:
|
||||||
|
- name: alpine-b6f790ae-5db2-4458-8375-59450f12f65b
|
||||||
|
container:
|
||||||
|
image: alpine:3.7
|
||||||
|
command: [sh, -c]
|
||||||
|
args: ['grep ''<meta name="description"'' /mnt/vol/tfc.html | awk -F ''content="'' ''{print $2}'' | awk -F ''"'' ''{print $1}''']
|
||||||
|
volumeMounts:
|
||||||
|
- name: workdir
|
||||||
|
mountPath: /mnt/vol
|
||||||
|
- name: curl-aa8d2265-9fe2-42c7-ba1f-46ea0da8e633
|
||||||
|
container:
|
||||||
|
image: curlimages/curl:7.88.1
|
||||||
|
args: [-SL, 'https://toulousefc.com', -o, /mnt/vol/tfc.hmtl]
|
||||||
|
volumeMounts:
|
||||||
|
- name: workdir
|
||||||
|
mountPath: /mnt/vol
|
||||||
|
- name: dag
|
||||||
|
dag:
|
||||||
|
tasks:
|
||||||
|
- name: curl-aa8d2265-9fe2-42c7-ba1f-46ea0da8e633
|
||||||
|
template: curl-aa8d2265-9fe2-42c7-ba1f-46ea0da8e633
|
||||||
|
- name: alpine-b6f790ae-5db2-4458-8375-59450f12f65b
|
||||||
|
template: alpine-b6f790ae-5db2-4458-8375-59450f12f65b
|
||||||
|
dependencies:
|
||||||
|
- curl-aa8d2265-9fe2-42c7-ba1f-46ea0da8e633
|
46
argo_workflows/spresis-aplaront_01_08_2024_182306.yml
Normal file
46
argo_workflows/spresis-aplaront_01_08_2024_182306.yml
Normal file
@ -0,0 +1,46 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Workflow
|
||||||
|
metadata:
|
||||||
|
generateName: oc-test-spresis-aplaront
|
||||||
|
spec:
|
||||||
|
entrypoint: dag
|
||||||
|
volumeClaimTemplates:
|
||||||
|
- metadata:
|
||||||
|
name: workdir
|
||||||
|
spec:
|
||||||
|
accessModes: [ReadWriteOnce]
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 1Gi
|
||||||
|
templates:
|
||||||
|
- name: "-0"
|
||||||
|
container:
|
||||||
|
image: ""
|
||||||
|
args: [""]
|
||||||
|
volumeMounts:
|
||||||
|
- name: workdir
|
||||||
|
mountPath: /mnt/vol
|
||||||
|
- name: curl-1
|
||||||
|
container:
|
||||||
|
image: curlimages/curl:7.88.1
|
||||||
|
args: [-SL, 'https://toulousefc.com', -o, /mnt/vol/tfc.hmtl]
|
||||||
|
volumeMounts:
|
||||||
|
- name: workdir
|
||||||
|
mountPath: /mnt/vol
|
||||||
|
- name: alpine-2
|
||||||
|
container:
|
||||||
|
image: alpine:3.7
|
||||||
|
command: [sh, -c]
|
||||||
|
args: ['grep ''<meta name="description"'' /mnt/vol/tfc.html | awk -F ''content="'' ''{print $2}'' | awk -F ''"'' ''{print $1}''']
|
||||||
|
volumeMounts:
|
||||||
|
- name: workdir
|
||||||
|
mountPath: /mnt/vol
|
||||||
|
- name: dag
|
||||||
|
dag:
|
||||||
|
tasks:
|
||||||
|
- name: '-'
|
||||||
|
template: '-'
|
||||||
|
- name: curl-0d565c87-50ae-4a73-843d-f8b2d4047772
|
||||||
|
template: curl-0d565c87-50ae-4a73-843d-f8b2d4047772
|
||||||
|
- name: alpine-2ce0323f-a85d-4b8b-a783-5280f48d634a
|
||||||
|
template: alpine-2ce0323f-a85d-4b8b-a783-5280f48d634a
|
3
conf/docker_ocmonitor_conf.json
Normal file
3
conf/docker_ocmonitor_conf.json
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
{
|
||||||
|
"oc-catalog": "https://oc-catalog:8087"
|
||||||
|
}
|
3
conf/local_ocmonitor_conf.json
Normal file
3
conf/local_ocmonitor_conf.json
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
{
|
||||||
|
"oc-catalog": "https://localhost:8087"
|
||||||
|
}
|
15
logger/logger.go
Normal file
15
logger/logger.go
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
package logger
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/rs/zerolog"
|
||||||
|
)
|
||||||
|
|
||||||
|
var Logger zerolog.Logger
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
output := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC3339}
|
||||||
|
Logger = zerolog.New(output).With().Timestamp().Logger()
|
||||||
|
}
|
53
models/http.go
Normal file
53
models/http.go
Normal file
@ -0,0 +1,53 @@
|
|||||||
|
package models
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/cookiejar"
|
||||||
|
"net/url"
|
||||||
|
)
|
||||||
|
|
||||||
|
type HttpQuery struct {
|
||||||
|
baseurl string
|
||||||
|
jar http.CookieJar
|
||||||
|
Cookies map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *HttpQuery) Init(url string) {
|
||||||
|
h.baseurl = url
|
||||||
|
h.jar, _ = cookiejar.New(nil)
|
||||||
|
h.Cookies = make(map[string]string)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *HttpQuery) Get(url string) ([]byte, error) {
|
||||||
|
client := &http.Client{Jar: h.jar}
|
||||||
|
resp, err := client.Get(h.baseurl + url)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// store received cookies
|
||||||
|
for _, cookie := range h.jar.Cookies(resp.Request.URL) {
|
||||||
|
h.Cookies[cookie.Name] = cookie.Value
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return body, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *HttpQuery) Post(url string, data url.Values) (*http.Response, error) {
|
||||||
|
client := &http.Client{Jar: h.jar}
|
||||||
|
resp, err := client.PostForm(h.baseurl+url, data)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// store received cookies
|
||||||
|
for _, cookie := range h.jar.Cookies(resp.Request.URL) {
|
||||||
|
h.Cookies[cookie.Name] = cookie.Value
|
||||||
|
}
|
||||||
|
return resp, err
|
||||||
|
}
|
74
models/schedule.go
Normal file
74
models/schedule.go
Normal file
@ -0,0 +1,74 @@
|
|||||||
|
package models
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"oc-monitor/logger"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Is duration really important ?
|
||||||
|
|
||||||
|
type Booking struct {
|
||||||
|
Start time.Time
|
||||||
|
Stop time.Time
|
||||||
|
Duration uint
|
||||||
|
Workflow string
|
||||||
|
}
|
||||||
|
|
||||||
|
type ScheduledBooking struct {
|
||||||
|
Bookings []Booking
|
||||||
|
Mu sync.Mutex
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s Booking) Equals(other Booking) bool {
|
||||||
|
return s.Workflow == other.Workflow && s.Start == other.Start && s.Stop == other.Stop
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sb *ScheduledBooking) AddSchedule(new_booking Booking){
|
||||||
|
if(!sb.scheduleAlreadyExists(new_booking)){
|
||||||
|
sb.Bookings = append(sb.Bookings,new_booking)
|
||||||
|
logger.Logger.Info().Msg("Updated list schedules : \n " + sb.String())
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
} else {
|
||||||
|
// Debug condition : delete once this feature is ready to be implemented
|
||||||
|
logger.Logger.Debug().Msg("Workflow received not added")
|
||||||
|
logger.Logger.Debug().Msg("current schedule contains")
|
||||||
|
for _, booking := range(sb.Bookings){
|
||||||
|
logger.Logger.Debug().Msg(booking.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sb *ScheduledBooking) GetListNames()(list_names []string ){
|
||||||
|
for _, schedule := range(sb.Bookings){
|
||||||
|
list_names = append(list_names, schedule.Workflow)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sb *ScheduledBooking) scheduleAlreadyExists(new_booking Booking) bool {
|
||||||
|
for _, booking := range(sb.Bookings){
|
||||||
|
if booking.Equals(new_booking){
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Booking) String() string {
|
||||||
|
return fmt.Sprintf("{workflow : %s , start_date : %s , stop_date : %s }", b.Workflow, b.Start.Format(time.RFC3339), b.Stop.Format(time.RFC3339))
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sb *ScheduledBooking) String() string {
|
||||||
|
var str string
|
||||||
|
for _, booking := range(sb.Bookings){
|
||||||
|
str += fmt.Sprintf("%s\n", booking.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
return str
|
||||||
|
}
|
40
models/template_models.go
Normal file
40
models/template_models.go
Normal file
@ -0,0 +1,40 @@
|
|||||||
|
package models
|
||||||
|
|
||||||
|
type Parameter struct {
|
||||||
|
Name string `yaml:"name,omitempty"`
|
||||||
|
Value string `yaml:"value,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Container struct {
|
||||||
|
Image string `yaml:"image"`
|
||||||
|
Command []string `yaml:"command,omitempty,flow"`
|
||||||
|
Args []string `yaml:"args,omitempty,flow"`
|
||||||
|
VolumeMounts []VolumeMount `yaml:"volumeMounts,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type VolumeMount struct {
|
||||||
|
Name string `yaml:"name"`
|
||||||
|
MountPath string `yaml:"mountPath"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Task struct {
|
||||||
|
Name string `yaml:"name"`
|
||||||
|
Template string `yaml:"template"`
|
||||||
|
Dependencies []string `yaml:"dependencies,omitempty"`
|
||||||
|
Arguments struct {
|
||||||
|
Parameters []Parameter `yaml:"parameters,omitempty"`
|
||||||
|
} `yaml:"arguments,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Dag struct {
|
||||||
|
Tasks []Task `yaml:"tasks,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Template struct {
|
||||||
|
Name string `yaml:"name"`
|
||||||
|
Inputs struct {
|
||||||
|
Parameters []Parameter `yaml:"parameters"`
|
||||||
|
} `yaml:"inputs,omitempty"`
|
||||||
|
Container Container `yaml:"container,omitempty"`
|
||||||
|
Dag Dag `yaml:"dag,omitempty"`
|
||||||
|
}
|
19
models/volume_models.go
Normal file
19
models/volume_models.go
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
package models
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
type VolumeClaimTemplate struct {
|
||||||
|
Metadata struct {
|
||||||
|
Name string `yaml:"name"`
|
||||||
|
} `yaml:"metadata"`
|
||||||
|
Spec VolumeSpec `yaml:"spec"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type VolumeSpec struct {
|
||||||
|
AccessModes []string `yaml:"accessModes,flow"`
|
||||||
|
Resources struct {
|
||||||
|
Requests struct {
|
||||||
|
Storage string `yaml:"storage"`
|
||||||
|
} `yaml:"requests"`
|
||||||
|
} `yaml:"resources"`
|
||||||
|
}
|
315
workflow_builder/argo_builder.go
Normal file
315
workflow_builder/argo_builder.go
Normal file
@ -0,0 +1,315 @@
|
|||||||
|
// A class that translates the informations held in the graph object
|
||||||
|
// via its lists of components into an argo file, using the a list of
|
||||||
|
// link ID to build the dag
|
||||||
|
|
||||||
|
package workflow_builder
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"oc-monitor/logger"
|
||||||
|
. "oc-monitor/models"
|
||||||
|
"os"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"cloud.o-forge.io/core/oc-lib/models/resource_model"
|
||||||
|
"cloud.o-forge.io/core/oc-lib/models/resources/workflow/graph"
|
||||||
|
"github.com/nwtgck/go-fakelish"
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ArgoBuilder struct {
|
||||||
|
graph graph.Graph
|
||||||
|
branches [][]int
|
||||||
|
Workflow Workflow
|
||||||
|
}
|
||||||
|
|
||||||
|
type Workflow struct {
|
||||||
|
ApiVersion string `yaml:"apiVersion"`
|
||||||
|
Kind string `yaml:"kind"`
|
||||||
|
Metadata struct {
|
||||||
|
GenerateName string `yaml:"generateName"`
|
||||||
|
} `yaml:"metadata"`
|
||||||
|
Spec Spec `yaml:"spec,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Spec struct {
|
||||||
|
Entrypoint string `yaml:"entrypoint"`
|
||||||
|
Arguments []Parameter `yaml:"arguments,omitempty"`
|
||||||
|
Volumes []VolumeClaimTemplate `yaml:"volumeClaimTemplates,omitempty"`
|
||||||
|
Templates []Template `yaml:"templates"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *ArgoBuilder) CreateDAG() (string, error) {
|
||||||
|
fmt.Println("list of branches : ", b.branches)
|
||||||
|
|
||||||
|
b.createTemplates()
|
||||||
|
b.createDAGstep()
|
||||||
|
b.createVolumes()
|
||||||
|
b.Workflow.Spec.Entrypoint = "dag"
|
||||||
|
|
||||||
|
b.Workflow.ApiVersion = "argoproj.io/v1alpha1"
|
||||||
|
b.Workflow.Kind = "Workflow"
|
||||||
|
random_name := generateWfName()
|
||||||
|
b.Workflow.Metadata.GenerateName = "oc-test-" + random_name
|
||||||
|
|
||||||
|
yamlified, err := yaml.Marshal(b.Workflow)
|
||||||
|
if err != nil {
|
||||||
|
logger.Logger.Error().Msg("Could not transform object to yaml file")
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Give a unique name to each argo file with its timestamp DD:MM:YYYY_hhmmss
|
||||||
|
current_timestamp := time.Now().Format("02_01_2006_150405")
|
||||||
|
file_name := random_name + "_" + current_timestamp + ".yml"
|
||||||
|
workflows_dir := "argo_workflows/"
|
||||||
|
err = os.WriteFile(workflows_dir+file_name, []byte(yamlified), 0660)
|
||||||
|
if err != nil {
|
||||||
|
logger.Logger.Error().Msg("Could not write the yaml file")
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return file_name, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *ArgoBuilder) createTemplates() {
|
||||||
|
|
||||||
|
for _, comp := range b.getProcessings() {
|
||||||
|
var command string
|
||||||
|
var args string
|
||||||
|
var env string
|
||||||
|
|
||||||
|
comp_res := comp.Processing
|
||||||
|
|
||||||
|
command = getStringValue(comp_res.AbstractResource, "command")
|
||||||
|
args = getStringValue(comp_res.AbstractResource, "args")
|
||||||
|
env = getStringValue(comp_res.AbstractResource, "env")
|
||||||
|
|
||||||
|
image_name := strings.Split(command, " ")[0] // TODO : decide where to store the image name, GUI or models.computing.Image
|
||||||
|
temp_container := Container{Image: image_name} // TODO : decide where to store the image name, GUI or models.computing.Image
|
||||||
|
temp_container.Command = getComputingCommands(command)
|
||||||
|
temp_container.Args = getComputingArgs(strings.Split(args, " "), command)
|
||||||
|
// Only for dev purpose,
|
||||||
|
input_names := getComputingEnvironmentName(strings.Split(env, " "))
|
||||||
|
|
||||||
|
var inputs_container []Parameter
|
||||||
|
for _, name := range input_names {
|
||||||
|
inputs_container = append(inputs_container, Parameter{Name: name})
|
||||||
|
}
|
||||||
|
|
||||||
|
argo_name := getArgoName(comp_res.GetName(), comp.ID)
|
||||||
|
new_temp := Template{Name: argo_name, Container: temp_container}
|
||||||
|
new_temp.Inputs.Parameters = inputs_container
|
||||||
|
new_temp.Container.VolumeMounts = append(new_temp.Container.VolumeMounts, VolumeMount{Name: "workdir", MountPath: "/mnt/vol"}) // TODO : replace this with a search of the storage / data source name
|
||||||
|
b.Workflow.Spec.Templates = append(b.Workflow.Spec.Templates, new_temp)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *ArgoBuilder) createDAGstep() {
|
||||||
|
|
||||||
|
new_dag := Dag{}
|
||||||
|
|
||||||
|
for _, comp := range b.getProcessings() {
|
||||||
|
|
||||||
|
comp_res := comp.Processing
|
||||||
|
|
||||||
|
env := getStringValue(comp_res.AbstractResource, "env")
|
||||||
|
unique_name := getArgoName(comp_res.GetName(), comp.ID)
|
||||||
|
step := Task{Name: unique_name, Template: unique_name}
|
||||||
|
comp_envs := getComputingEnvironment(strings.Split(env, " "))
|
||||||
|
|
||||||
|
for name, value := range comp_envs {
|
||||||
|
step.Arguments.Parameters = append(step.Arguments.Parameters, Parameter{Name: name, Value: value})
|
||||||
|
}
|
||||||
|
|
||||||
|
// retrieves the name (computing.name-computing.ID)
|
||||||
|
step.Dependencies = b.getDependency(comp.ID) // Error : we use the component ID instead of the GraphItem ID -> store objects
|
||||||
|
|
||||||
|
new_dag.Tasks = append(new_dag.Tasks, step)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
b.Workflow.Spec.Templates = append(b.Workflow.Spec.Templates, Template{Name: "dag", Dag: new_dag})
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *ArgoBuilder) createVolumes() {
|
||||||
|
// For testing purposes we only declare one volume, mounted in each computing
|
||||||
|
new_volume := VolumeClaimTemplate{}
|
||||||
|
new_volume.Metadata.Name = "workdir"
|
||||||
|
new_volume.Spec.AccessModes = []string{"ReadWriteOnce"}
|
||||||
|
new_volume.Spec.Resources.Requests.Storage = "1Gi"
|
||||||
|
|
||||||
|
b.Workflow.Spec.Volumes = append(b.Workflow.Spec.Volumes, new_volume)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *ArgoBuilder) getDependency(current_computing_id string) (dependencies []string) {
|
||||||
|
var dependencies_id []string
|
||||||
|
|
||||||
|
for _, link := range b.graph.Links {
|
||||||
|
source := b.graph.Items[link.Source.ID].Processing // Instead of searching for the AbstractResource we load the Processing pointer and test if it's nil to know if the item is a processing
|
||||||
|
if current_computing_id == link.Destination.ID && source != nil && !slices.Contains(dependencies_id, link.Source.ID) {
|
||||||
|
dependencies_id = append(dependencies_id, link.Source.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, dependency := range dependencies_id {
|
||||||
|
source := b.graph.Items[dependency].Processing
|
||||||
|
dependency_name := getArgoName(source.GetName(), dependency)
|
||||||
|
dependencies = append(dependencies, dependency_name)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// func (b *ArgoBuilder) componentInBranch(component_id string, branch []string) bool {
|
||||||
|
// for _, link := range branch {
|
||||||
|
// if b.graph.Links[link].Source == component_id || b.graph.Links[link].Destination == component_id {
|
||||||
|
// return true
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// return false
|
||||||
|
// }
|
||||||
|
|
||||||
|
// func (b *ArgoBuilder) findPreviousComputing(computing_id string, branch []string, index int) string {
|
||||||
|
|
||||||
|
// for i := index; i >= 0 ; i-- {
|
||||||
|
// previousLink := b.graph.Links[branch[i]]
|
||||||
|
|
||||||
|
// if previousLink.Source != computing_id && b.graph.GetComponentType(previousLink.Source) == "computing"{
|
||||||
|
// name := getArgoName(b.graph.getComponentName(previousLink.Source),previousLink.Source)
|
||||||
|
// return name
|
||||||
|
// }
|
||||||
|
// if previousLink.Destination != computing_id && b.graph.GetComponentType(previousLink.Destination) == "computing"{
|
||||||
|
// name := getArgoName(b.graph.getComponentName(previousLink.Destination),previousLink.Destination)
|
||||||
|
// return name
|
||||||
|
// }
|
||||||
|
|
||||||
|
// }
|
||||||
|
// return ""
|
||||||
|
// }
|
||||||
|
|
||||||
|
func getComputingCommands(user_input string) (list_command []string) {
|
||||||
|
user_input = removeImageName(user_input)
|
||||||
|
if len(user_input) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
list_command = strings.Split(user_input, " ")
|
||||||
|
for i := range list_command {
|
||||||
|
list_command[i] = list_command[i]
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func getComputingArgs(user_input []string, command string) (list_args []string) {
|
||||||
|
if len(user_input) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// quickfix that might need improvement
|
||||||
|
if strings.Contains(command, "sh -c") {
|
||||||
|
list_args = append(list_args, strings.Join(user_input, " "))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, arg := range user_input {
|
||||||
|
list_args = append(list_args, arg)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Currently implements code to overcome problems in data structure
|
||||||
|
func getComputingEnvironment(user_input []string) (map_env map[string]string) {
|
||||||
|
|
||||||
|
is_empty := len(user_input) == 0
|
||||||
|
is_empty_string := len(user_input) == 1 && user_input[0] == ""
|
||||||
|
|
||||||
|
if is_empty || is_empty_string {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(user_input) == 1 {
|
||||||
|
user_input = strings.Split(user_input[0], ",")
|
||||||
|
}
|
||||||
|
|
||||||
|
map_env = make(map[string]string, 0)
|
||||||
|
|
||||||
|
for _, str := range user_input {
|
||||||
|
new_pair := strings.Split(str, "=")
|
||||||
|
|
||||||
|
if len(new_pair) != 2 {
|
||||||
|
logger.Logger.Error().Msg("Error extracting the environment variable from " + str)
|
||||||
|
panic(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
map_env[new_pair[0]] = new_pair[1]
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func getComputingEnvironmentName(user_input []string) (list_names []string) {
|
||||||
|
env_map := getComputingEnvironment(user_input)
|
||||||
|
for name := range env_map {
|
||||||
|
list_names = append(list_names, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func generateWfName() (Name string) {
|
||||||
|
Name = fakelish.GenerateFakeWord(5, 8) + "-" + fakelish.GenerateFakeWord(5, 8)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func getArgoName(raw_name string, component_id string) (formatedName string) {
|
||||||
|
formatedName = strings.ReplaceAll(raw_name, " ", "-")
|
||||||
|
formatedName += "-" + component_id
|
||||||
|
formatedName = strings.ToLower(formatedName)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func printYAML(data interface{}) {
|
||||||
|
yamlData, err := yaml.Marshal(data)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Error marshalling YAML: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Println(string(yamlData))
|
||||||
|
}
|
||||||
|
|
||||||
|
func removeImageName(user_input string) string {
|
||||||
|
// First command is the name of the container for now
|
||||||
|
if len(strings.Split(user_input, " ")) == 1 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
slice_input := strings.Split(user_input, " ")
|
||||||
|
new_slice := slice_input[1:]
|
||||||
|
user_input = strings.Join(new_slice, " ")
|
||||||
|
|
||||||
|
return user_input
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return the graphItem containing a Processing resource, so that we have access to the ID of the graphItem in order to identify it in the links
|
||||||
|
func (b *ArgoBuilder) getProcessings() (list_computings []graph.GraphItem) {
|
||||||
|
for _, item := range b.graph.Items {
|
||||||
|
if item.Processing != nil {
|
||||||
|
list_computings = append(list_computings, item)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func getStringValue(comp resource_model.AbstractResource, key string) string {
|
||||||
|
if res := comp.GetModelValue(key); res != nil {
|
||||||
|
return res.(string)
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
246
workflow_builder/graph.go
Normal file
246
workflow_builder/graph.go
Normal file
@ -0,0 +1,246 @@
|
|||||||
|
package workflow_builder
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"maps"
|
||||||
|
|
||||||
|
"oc-monitor/logger"
|
||||||
|
models "oc-monitor/models"
|
||||||
|
|
||||||
|
oclib "cloud.o-forge.io/core/oc-lib"
|
||||||
|
"cloud.o-forge.io/core/oc-lib/models/resources/workflow/graph"
|
||||||
|
workflow "cloud.o-forge.io/core/oc-lib/models/workflow"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
type WorflowDB struct {
|
||||||
|
workflow_name string // used to test if the graph has been instatiated, private so can only be set by a graph's method
|
||||||
|
graph *graph.Graph
|
||||||
|
links map[int]graph.GraphLink
|
||||||
|
ws models.HttpQuery
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create the obj!ects from the mxgraphxml stored in the workflow given as a parameter
|
||||||
|
func (w *WorflowDB) LoadFrom(workflow_id string) error {
|
||||||
|
|
||||||
|
new_wf, err := w.getWorkflow(workflow_id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
w.graph = new_wf.Graph
|
||||||
|
w.links = w.getLinks()
|
||||||
|
|
||||||
|
w.workflow_name = new_wf.Name
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Use oclib to retrieve the graph contained in the workflow referenced
|
||||||
|
func (w *WorflowDB) getWorkflow( workflow_id string) (workflow *workflow.Workflow, err error) {
|
||||||
|
|
||||||
|
lib_data := oclib.LoadOne(oclib.LibDataEnum(oclib.WORKFLOW),workflow_id)
|
||||||
|
if lib_data.Code != 200 {
|
||||||
|
logger.Logger.Error().Msg("Error loading the graph")
|
||||||
|
return workflow, errors.New(lib_data.Err)
|
||||||
|
}
|
||||||
|
|
||||||
|
new_wf := lib_data.ToWorkflow()
|
||||||
|
if new_wf == nil {
|
||||||
|
logger.Logger.Error().Msg("WorflowDB object is empty for " + workflow_id )
|
||||||
|
return workflow, errors.New("WorflowDB can't be empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
return new_wf, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
func (w *WorflowDB) getLinks() map[int]graph.GraphLink {
|
||||||
|
links := make(map[int]graph.GraphLink)
|
||||||
|
for i, link := range(w.graph.Links) {
|
||||||
|
links[i] = link
|
||||||
|
}
|
||||||
|
|
||||||
|
return links
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *WorflowDB) ExportToArgo() (string, error) {
|
||||||
|
if len(w.workflow_name) == 0 || &w.graph == nil {
|
||||||
|
return "",fmt.Errorf("can't export a graph that has not been loaded yet")
|
||||||
|
}
|
||||||
|
end_links := make(map[int]graph.GraphLink)
|
||||||
|
|
||||||
|
for i, link := range w.links {
|
||||||
|
if (!w.isDCLink(i) && !w.isSource(link.Destination.ID,i)){
|
||||||
|
end_links[i] = link
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// index_list := make([]int, len(w.links))
|
||||||
|
// list_branches := make([][]string,0)
|
||||||
|
list_branches := w.getListBranches(end_links, nil,nil)
|
||||||
|
|
||||||
|
// for _, branch := range list_branches{
|
||||||
|
// str := ""
|
||||||
|
// for _, link := range branch{
|
||||||
|
// str = str + " --> " + w.getComponentName(w.graph.Links[link].Source) + " linked with " + w.getComponentName(w.links[link].Destination)
|
||||||
|
// }
|
||||||
|
|
||||||
|
// fmt.Println(str)
|
||||||
|
// }
|
||||||
|
|
||||||
|
fmt.Println("Identified branches : ", list_branches)
|
||||||
|
argo_builder := ArgoBuilder{graph : *w.graph, branches: list_branches}
|
||||||
|
filename, err := argo_builder.CreateDAG()
|
||||||
|
if err != nil {
|
||||||
|
logger.Logger.Error().Msg("Could not create the argo file for " + w.workflow_name)
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return filename, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return a list containing the IDs of each link that make up a branch in the graph
|
||||||
|
func (w *WorflowDB) getListBranches(end_links map[int]graph.GraphLink, unvisited_links_list map[int]graph.GraphLink, current_branch []int) (list_branches [][]int) {
|
||||||
|
|
||||||
|
if current_branch == nil {
|
||||||
|
current_branch = make([]int, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
if unvisited_links_list == nil {
|
||||||
|
unvisited_links_list = make(map[int]graph.GraphLink,len(w.graph.Links))
|
||||||
|
maps.Copy(unvisited_links_list,w.links)
|
||||||
|
fmt.Println(unvisited_links_list)
|
||||||
|
}
|
||||||
|
|
||||||
|
for link_id, _ := range end_links {
|
||||||
|
j := link_id
|
||||||
|
new_branches := make([][]int,0)
|
||||||
|
|
||||||
|
previous_index := w.getPreviousLink(j, unvisited_links_list)
|
||||||
|
if len(previous_index) == 0 {
|
||||||
|
list_branches = append(list_branches, []int{link_id})
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, id_link := range previous_index {
|
||||||
|
current_branch = append([]int{link_id},current_branch...)
|
||||||
|
delete(unvisited_links_list, link_id)
|
||||||
|
// create a new branch for each previous link, appending the current path to this node to the created branch
|
||||||
|
new_end_link := make(map[int]graph.GraphLink,0)
|
||||||
|
new_end_link[id_link] = w.links[id_link]
|
||||||
|
new_branches = w.getListBranches(new_end_link,unvisited_links_list,current_branch)
|
||||||
|
|
||||||
|
for _, new_branch := range new_branches{
|
||||||
|
current_branch = append(new_branch,link_id)
|
||||||
|
list_branches = append(list_branches, current_branch)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *WorflowDB) ExportToHelm(id string) error {
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return if it exists a link where Destination is the same as comp_id
|
||||||
|
func (w *WorflowDB) isDestination(comp_id string,link_id int) bool {
|
||||||
|
|
||||||
|
for i, link := range w.links{
|
||||||
|
if(i !=link_id && link.Destination.ID == comp_id){
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return if it exists a link where Source is the same as comp_id
|
||||||
|
func (w *WorflowDB) isSource(comp_id string, link_id int) bool {
|
||||||
|
|
||||||
|
for i, link := range w.links{
|
||||||
|
if(i !=link_id && link.Source.ID == comp_id && !w.isDCLink(i)){
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// Returns an index number if their is a link in w.links
|
||||||
|
// with the same Destination id that the Source id in w.links[linkIndex]
|
||||||
|
// or nil if not
|
||||||
|
|
||||||
|
func (w *WorflowDB) getPreviousLink(link_id int,map_link map[int]graph.GraphLink) (previous_id []int) {
|
||||||
|
for k, link := range map_link{
|
||||||
|
if(k != link_id && link.Destination == w.links[link_id].Source){
|
||||||
|
previous_id = append(previous_id, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
// returns either computing, data or storage
|
||||||
|
func GetComponentType(component_id string) string {
|
||||||
|
if libdata := oclib.LoadOne(oclib.LibDataEnum(oclib.PROCESSING_RESOURCE),component_id); libdata.Code == 200{
|
||||||
|
return "computing"
|
||||||
|
}
|
||||||
|
|
||||||
|
if libdata := oclib.LoadOne(oclib.LibDataEnum(oclib.DATA_RESOURCE),component_id); libdata.Code == 200{
|
||||||
|
return "data"
|
||||||
|
}
|
||||||
|
|
||||||
|
if libdata := oclib.LoadOne(oclib.LibDataEnum(oclib.STORAGE_RESOURCE),component_id); libdata.Code == 200{
|
||||||
|
return "storage"
|
||||||
|
}
|
||||||
|
|
||||||
|
if libdata := oclib.LoadOne(oclib.LibDataEnum(oclib.DATACENTER_RESOURCE),component_id); libdata.Code == 200{
|
||||||
|
return "datacenter"
|
||||||
|
}
|
||||||
|
|
||||||
|
if libdata := oclib.LoadOne(oclib.LibDataEnum(oclib.WORKFLOW_RESOURCE),component_id); libdata.Code == 200{
|
||||||
|
return "workflow"
|
||||||
|
}
|
||||||
|
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// Returns a slice of id, in case the link is made of twice the same type of component
|
||||||
|
|
||||||
|
func (w *WorflowDB) getComponentByType(compType string, link graph.GraphLink) (ids []string){
|
||||||
|
if(GetComponentType(link.Source.ID) == compType){
|
||||||
|
ids = append(ids, link.Source.ID)
|
||||||
|
}
|
||||||
|
if(GetComponentType(link.Destination.ID) == compType){
|
||||||
|
ids = append(ids, link.Destination.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *WorflowDB) isDCLink(link_id int) bool {
|
||||||
|
link := w.links[link_id]
|
||||||
|
|
||||||
|
dest := w.graph.Items[link.Destination.ID]
|
||||||
|
dest_id := dest.GetAbstractRessource().GetID()
|
||||||
|
|
||||||
|
source := w.graph.Items[link.Source.ID]
|
||||||
|
source_id := source.GetAbstractRessource().GetID()
|
||||||
|
|
||||||
|
return IsDatacenter(dest_id) || IsDatacenter(source_id)
|
||||||
|
}
|
||||||
|
|
||||||
|
func IsDatacenter(id string) bool {
|
||||||
|
resource := oclib.LoadOne(oclib.LibDataEnum(oclib.DATACENTER_RESOURCE),id)
|
||||||
|
|
||||||
|
return resource.Code == 200
|
||||||
|
}
|
10
workflow_builder/graph_tests.go
Normal file
10
workflow_builder/graph_tests.go
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
package workflow_builder
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGetGraph(t *testing.T){
|
||||||
|
w := WorflowDB{}
|
||||||
|
w.LoadFrom("test-log")
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user