Compare commits

..

No commits in common. "main" and "feature/schedule_executions" have entirely different histories.

29 changed files with 1243 additions and 545 deletions

18
.vscode/launch.json vendored
View File

@ -1,18 +0,0 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Launch Package",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${fileDirname}",
"env": {
"MONITOR_METHOD" : "local"
}
}
]
}

View File

@ -4,22 +4,15 @@ ENV DOCKER_ENVIRONMENT=true
WORKDIR /app
COPY . .
COPY conf/docker_scheduler.json /etc/oc/scheduler.json
RUN go build .
FROM oc-monitord:latest AS monitord
FROM argoproj/argocd:latest
ENV MONITORD_PATH = "./oc-monitord"
FROM golang:alpine
WORKDIR /app
COPY conf/docker_schedulerd.json /etc/oc/schedulerd.json
COPY --from=builder /app/oc-scheduler .
COPY conf/docker_scheduler.json /etc/oc/scheduler.json
COPY --from=monitord /app/oc-monitord .
COPY --from=builder /app/oc-schedulerd .
COPY conf/docker_schedulerd.json /etc/oc/schedulerd.json
ENTRYPOINT ["/app/oc-schedulerd"]
ENTRYPOINT ["/app/oc-scheduler"]

View File

@ -8,42 +8,48 @@ import (
)
type Config struct {
MonitorPath string
MongoUrl string
DBName string
Logs string
LokiUrl string
NatsUrl string
OcCatalogUrl string
Logs string
LokiUrl string
}
var instance *Config
var once sync.Once
const defaultConfigFile = "/etc/oc/schedulerd.json"
const defaultConfigFile = "/etc/oc/scheduler.json"
const localConfigFile = "./conf/local_scheduler.json"
func init() {
func init(){
configFile := ""
var o *onion.Onion
l3 := onion.NewEnvLayerPrefix("_", "OCSCHEDULERD_")
l3 := onion.NewEnvLayerPrefix("_", "OCSCHEDULER_")
l2, err := onion.NewFileLayer(defaultConfigFile, nil)
if err == nil {
logs.Info("Config file found : " + defaultConfigFile)
configFile = defaultConfigFile
}
if configFile == "" || l2 == nil {
l1, err := onion.NewFileLayer(localConfigFile, nil)
if err == nil {
logs.Info("Local config file found " + localConfigFile + ", overriding default file")
configFile = localConfigFile
}
if configFile == "" {
logs.Info("No config file found, using env")
o = onion.New(l3)
} else {
} else if l1 == nil && l2 == nil {
o = onion.New(l1, l2, l3)
} else if l1 == nil {
o = onion.New(l2, l3)
} else if l2 == nil {
o = onion.New(l1, l3)
}
GetConfig().MonitorPath = o.GetStringDefault("MONITORD_PATH", "../oc-monitord/oc-monitord")
GetConfig().Logs = o.GetStringDefault("LOG_LEVEL", "info")
GetConfig().LokiUrl = o.GetStringDefault("LOKI_URL", "http://127.0.0.1:3100")
GetConfig().NatsUrl = o.GetStringDefault("NATS_URL", "http://127.0.0.1:4222")
GetConfig().MongoUrl = o.GetStringDefault("MONGO_URL", "mongodb://127.0.0.1:27017")
GetConfig().DBName = o.GetStringDefault("MONGO_DATABASE", "DC_myDC")
GetConfig().OcCatalogUrl = o.GetStringDefault("oc-catalog", "https://localhost:49618")
GetConfig().Logs = o.GetStringDefault("loglevel", "info")
GetConfig().LokiUrl = o.GetStringDefault("loki_url","http://127.0.0.1:3100")
}
func GetConfig() *Config {

View File

@ -0,0 +1,4 @@
{
"oc-catalog" : "http://oc-catalog:49618/",
"loki_url" : "http://192.168.1.18:3100"
}

View File

@ -1,6 +0,0 @@
{
"LOKI_URL" : "http://loki:3100",
"MONGO_URL":"mongodb://mongo:27017/",
"MONGO_DATABASE":"DC_myDC",
"NATS_URL": "nats://nats:4222"
}

View File

@ -1,8 +0,0 @@
datasources:
- name: Loki
type: loki
access: proxy
url: http://loki:3100
isDefault: true
jsonData:
httpMethod: POST

View File

@ -0,0 +1,4 @@
{
"oc-catalog" : "http://localhost:49618/",
"logs" : ""
}

View File

@ -1,67 +1,3 @@
package daemons
// type manifestValues struct{
// ARGO_FILE string
// LOKI_URL string
// CONTAINER_NAME string
// }
// manifest, err := em.CreateManifest(booking, argo_file_path)
// if err != nil {
// logger.Logger.Error().Msg("Could not create manifest " + err.Error())
// }
// // launch a pod that contains oc-monitor, give it the name of the workflow
// cmd := exec.Command("kubectl","apply","-f", "manifests/"+manifest)
// output , err := cmd.CombinedOutput()
// if err != nil {
// logger.Logger.Error().Msg("failed to create new pod for " + booking.Workflow + " :" + err.Error())
// return
// }
// func (*ExecutionManager) CreateManifest(booking models.Booking, argo_file string) (manifest_filepath string, err error) {
// var filled_template bytes.Buffer
// manifest_file, err := os.ReadFile("conf/monitor_pod_template.yml")
// if err != nil {
// logger.Logger.Error().Msg("Could not open the k8s template file for " + booking.Workflow)
// return "", err
// }
// container_name := getContainerName(argo_file)
// tmpl, err := template.New("manifest_template").Parse(string(manifest_file))
// if err != nil {
// logger.Logger.Error().Msg(err.Error())
// return "", err
// }
// manifest_data := manifestValues{
// ARGO_FILE: argo_file,
// LOKI_URL: conf.GetConfig().Logs,
// CONTAINER_NAME: container_name,
// }
// err = tmpl.Execute(&filled_template, manifest_data)
// if err != nil {
// logger.Logger.Error().Msg("Could not complete manifest template for " + booking.Workflow)
// return "", err }
// manifest_filepath = booking.Workflow + "_manifest.yaml"
// err = os.WriteFile("manifests/" + manifest_filepath, filled_template.Bytes(),0644)
// if err != nil {
// logger.Logger.Error().Msg("Could not write the YAML file for " + booking.Workflow + "'s manifest")
// return "", err
// }
// return manifest_filepath, nil
// }
// func getContainerName(argo_file string) string {
// regex := "([a-zA-Z]+-[a-zA-Z]+)"
// re := regexp.MustCompile(regex)
// container_name := re.FindString(argo_file)
// return container_name
// }
// copy/transfer the argo_file to the created pod

View File

@ -1,46 +1,35 @@
package daemons
import (
"fmt"
"oc-schedulerd/conf"
"os/exec"
import "oc-scheduler/logger"
"github.com/rs/zerolog"
)
type LocalMonitor struct {
LokiURL string
KubeURL string
ExecutionID string
Duration int
Logger zerolog.Logger
type LocalMonitor struct{
LokiURL string
KubeURL string
ArgoFile string
}
func (lm *LocalMonitor) LaunchLocalMonitor() {
if lm.LokiURL == "" || lm.KubeURL == "" || lm.ExecutionID == "" {
lm.Logger.Error().Msg("Missing parameter in LocalMonitor")
func (lm *LocalMonitor) LaunchLocalMonitor (){
if (lm.LokiURL == "" || lm.KubeURL == "" || lm.ArgoFile == ""){
logger.Logger.Error().Msg("Missing parameter in LocalMonitor")
}
// For dev purposes, in prod KubeURL must be a kube API's URL
if lm.KubeURL != "localhost" {
lm.execRemoteKube()
} else {
lm.execLocalKube()
if(lm.KubeURL == "localhost"){
lm.ExecLocalKube()
} else{
lm.ExecRemoteKube()
}
}
func (lm *LocalMonitor) execLocalKube() {
args := []string{"-e", lm.ExecutionID, "-u", lm.LokiURL, "-m", conf.GetConfig().MongoUrl, "-d", conf.GetConfig().DBName}
if lm.Duration > 0 {
args = append(args, "-t", fmt.Sprintf("%d", lm.Duration))
}
cmd := exec.Command(conf.GetConfig().MonitorPath, args...)
fmt.Printf("Command : %v\n", cmd)
err := cmd.Start()
if err != nil {
lm.Logger.Error().Msg("Could not start oc-monitor for " + lm.ExecutionID + " : " + err.Error())
}
func (lm *LocalMonitor) ExecLocalKube (){
// kube_url := ""
}
// TODO : implement this
func (lm *LocalMonitor) execRemoteKube() {}
func (lm *LocalMonitor) ExecRemoteKube (){
}
func (lm *LocalMonitor) todo (){
}

View File

@ -1,60 +1,155 @@
package daemons
import (
"oc-schedulerd/conf"
"bytes"
"oc-scheduler/conf"
"oc-scheduler/logger"
"oc-scheduler/models"
"oc-scheduler/workflow_builder"
"os"
"os/exec"
"regexp"
"text/template"
"time"
oclib "cloud.o-forge.io/core/oc-lib"
workflow_execution "cloud.o-forge.io/core/oc-lib/models/workflow_execution"
)
var Bookings = ScheduledBooking{Bookings: []*workflow_execution.WorkflowExecution{}}
type ExecutionManager struct {
bookings *models.ScheduledBooking
executions []models.Booking
}
type ExecutionManager struct{}
type manifestValues struct{
ARGO_FILE string
LOKI_URL string
CONTAINER_NAME string
}
func (em *ExecutionManager) SetBookings(b *models.ScheduledBooking){
em.bookings = b
}
// Loop every second on the booking's list and move the booking that must start to a new list
// that will be looped over to start them
func (em *ExecutionManager) RetrieveNextExecutions() {
logger := oclib.GetLogger()
for {
logger.Debug().Msg("New loop")
Bookings.Mu.Lock()
if len(Bookings.Bookings) > 0 {
bookings := Bookings.Bookings
for i := len(bookings) - 1; i >= 0; i-- {
if bookings[i].ExecDate.Before(time.Now().UTC()) {
logger.Info().Msg("Will execute " + bookings[i].UUID + " soon")
func (em *ExecutionManager) RetrieveNextExecutions(){
if(em.bookings == nil){
logger.Logger.Fatal().Msg("booking has not been set in the exection manager")
}
for(true){
logger.Logger.Debug().Msg("New loop")
em.bookings.Mu.Lock()
bookings := em.bookings.Bookings
if (len(bookings) > 0){
for i := len( bookings) - 1 ; i >= 0 ; i--{
logger.Logger.Debug().Msg("It should start at " + bookings[i].Start.String() + " and it is now " + time.Now().UTC() .String())
if (bookings[i].Start.Before(time.Now().UTC())){
logger.Logger.Info().Msg("Will execute " + bookings[i].Workflow + " soon")
go em.executeBooking(bookings[i])
Bookings.Bookings = append(bookings[:i], bookings[i+1:]...)
bookings = append(bookings[:i], bookings[i+1:]...)
em.bookings.Bookings = bookings
}
}
}
Bookings.Mu.Unlock()
}
em.bookings.Mu.Unlock()
time.Sleep(time.Second)
}
}
func (em *ExecutionManager) executeBooking(booking *workflow_execution.WorkflowExecution) {
// start execution
// create the yaml that describes the pod : filename, path/url to Loki
exec_method := os.Getenv("MONITOR_METHOD")
logger := oclib.GetLogger()
if exec_method == "k8s" {
logger.Error().Msg("TODO : executing oc-monitor in a k8s")
} else {
logger.Debug().Msg("Executing oc-monitor localy")
duration := 0
if booking.EndDate != nil && booking.ExecDate != nil {
duration = int(booking.EndDate.Sub(*booking.ExecDate).Seconds())
}
monitor := LocalMonitor{
Logger: logger,
Duration: duration,
LokiURL: conf.GetConfig().LokiUrl,
KubeURL: "localhost",
ExecutionID: booking.UUID,
}
monitor.LaunchLocalMonitor()
func (em *ExecutionManager) executeBooking(booking models.Booking){
// create argo
new_graph := workflow_builder.Graph{}
err := new_graph.LoadFrom(booking.Workflow)
if err != nil {
logger.Logger.Error().Msg("Could not retrieve workflow " + booking.Workflow + " from oc-catalog API")
}
argo_file_path, err := new_graph.ExportToArgo()
if err != nil {
logger.Logger.Error().Msg("Could not create the Argo file for " + booking.Workflow )
logger.Logger.Error().Msg(err.Error())
}
logger.Logger.Debug().Msg("Created :" + argo_file_path)
// start execution
// create the yaml that describes the pod : filename, path/url to Loki
manifest, err := em.CreateManifest(booking, argo_file_path)
if err != nil {
logger.Logger.Error().Msg("Could not create manifest " + err.Error())
}
// launch a pod that contains oc-monitor, give it the name of the workflow
cmd := exec.Command("kubectl","apply","-f", "manifests/"+manifest)
output , err := cmd.CombinedOutput()
if err != nil {
logger.Logger.Error().Msg("failed to create new pod for " + booking.Workflow + " :" + err.Error())
return
}
logger.Logger.Debug().Msg("Result from kubectl apply : " + string(output))
// Transfer the argo file to the pod
cmd = exec.Command("kubectl","cp", "argo_workflows/" + argo_file_path, "pods/test-monitor:/app/workflows", "-c", "oc-monitor-" + getContainerName(argo_file_path))
output, err = cmd.CombinedOutput()
if err != nil {
logger.Logger.Error().Msg("failed to copy argo file to " + booking.Workflow + " :" + err.Error())
return
}
logger.Logger.Debug().Msg("Result from kubectl cp : " + string(output))
}
func (*ExecutionManager) CreateManifest(booking models.Booking, argo_file string) (manifest_filepath string, err error) {
var filled_template bytes.Buffer
manifest_file, err := os.ReadFile("conf/monitor_pod_template.yml")
if err != nil {
logger.Logger.Error().Msg("Could not open the k8s template file for " + booking.Workflow)
return "", err
}
container_name := getContainerName(argo_file)
tmpl, err := template.New("manifest_template").Parse(string(manifest_file))
if err != nil {
logger.Logger.Error().Msg(err.Error())
return "", err
}
manifest_data := manifestValues{
ARGO_FILE: argo_file,
LOKI_URL: conf.GetConfig().Logs,
CONTAINER_NAME: container_name,
}
err = tmpl.Execute(&filled_template, manifest_data)
if err != nil {
logger.Logger.Error().Msg("Could not complete manifest template for " + booking.Workflow)
return "", err }
manifest_filepath = booking.Workflow + "_manifest.yaml"
err = os.WriteFile("manifests/" + manifest_filepath, filled_template.Bytes(),0644)
if err != nil {
logger.Logger.Error().Msg("Could not write the YAML file for " + booking.Workflow + "'s manifest")
return "", err
}
return manifest_filepath, nil
}
func getContainerName(argo_file string) string {
regex := "([a-zA-Z]+-[a-zA-Z]+)"
re := regexp.MustCompile(regex)
container_name := re.FindString(argo_file)
return container_name
}

View File

@ -3,147 +3,121 @@ package daemons
import (
"encoding/json"
"fmt"
"oc-schedulerd/conf"
"sync"
"net/url"
"time"
oclib "cloud.o-forge.io/core/oc-lib"
"cloud.o-forge.io/core/oc-lib/dbs"
"cloud.o-forge.io/core/oc-lib/models/workflow_execution"
"cloud.o-forge.io/core/oc-lib/tools"
"oc-scheduler/logger"
"oc-scheduler/models"
"github.com/nats-io/nats.go"
"github.com/rs/zerolog"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type ScheduledBooking struct {
Bookings []*workflow_execution.WorkflowExecution
Mu sync.Mutex
}
func (sb *ScheduledBooking) DeleteSchedules(workflow_id string) {
toNotDelete := []*workflow_execution.WorkflowExecution{}
for _, b := range sb.Bookings {
if b.WorkflowID != workflow_id {
toNotDelete = append(toNotDelete, b)
}
}
Bookings.Mu.Lock()
defer Bookings.Mu.Unlock()
sb.Bookings = toNotDelete
}
func (sb *ScheduledBooking) AddSchedules(new_bookings []*workflow_execution.WorkflowExecution, logger zerolog.Logger) {
Bookings.Mu.Lock()
defer Bookings.Mu.Unlock()
for _, exec := range new_bookings {
if !sb.execIsSet(exec) {
sb.Bookings = append(sb.Bookings, exec)
}
}
}
func (sb *ScheduledBooking) execIsSet(exec *workflow_execution.WorkflowExecution) bool {
for _, b := range sb.Bookings {
if b.Equals(exec) {
return true
}
}
return false
}
// NATS daemon listens to subject " workflowsUpdate "
// workflowsUpdate messages must be formatted following this pattern '{"workflow" : "", "start_date" : "", "stop_date" : "" }'
type ScheduleManager struct {
Logger zerolog.Logger
Api_url string
bookings *models.ScheduledBooking
ws models.HttpQuery
}
func (s *ScheduleManager) SetBookings(b *models.ScheduledBooking){
s.bookings = b
}
// Goroutine listening to a NATS server for updates
// on workflows' scheduling. Messages must contain
// workflow execution ID, to allow retrieval of execution infos
func (s *ScheduleManager) ListenNATS() {
nc, err := nats.Connect(conf.GetConfig().NatsUrl)
if err != nil {
s.Logger.Error().Msg("Could not connect to NATS")
return
// on workflows' scheduling. Messages must contain
// workflow's name, start_date and stop_date while there
// is no way to get scheduling infos for a specific workflow
func (s *ScheduleManager) ListenForWorkflowSubmissions(){
if(s.bookings == nil){
logger.Logger.Fatal().Msg("booking has not been set in the schedule manager")
}
nc, _ := nats.Connect(nats.DefaultURL)
defer nc.Close()
var wg sync.WaitGroup
wg.Add(2)
go s.listenForChange(nc, tools.REMOVE.GenerateKey(oclib.WORKFLOW.String()), true, wg)
go s.listenForChange(nc, tools.CREATE.GenerateKey(oclib.WORKFLOW.String()), false, wg)
wg.Wait()
}
// Goroutine listening to a NATS server for updates
// on workflows' scheduling. Messages must contain
// workflow execution ID, to allow retrieval of execution infos
func (s *ScheduleManager) listenForChange(nc *nats.Conn, chanName string, delete bool, wg sync.WaitGroup) {
defer wg.Done()
ch := make(chan *nats.Msg, 64)
fmt.Println("Listening to " + chanName)
subs, err := nc.ChanSubscribe(chanName, ch)
subs , err := nc.ChanSubscribe("workflowsUpdate", ch)
if err != nil {
s.Logger.Error().Msg("Error listening to NATS : " + err.Error())
logger.Logger.Fatal().Msg("Error listening to NATS")
}
defer subs.Unsubscribe()
for msg := range ch {
map_mess := map[string]string{}
json.Unmarshal(msg.Data, &map_mess)
str := "new"
if delete {
str = "deleted"
for msg := range(ch){
fmt.Println("Waiting...")
map_mess := retrieveMapFromSub(msg.Data)
s.bookings.Mu.Lock()
start, err := time.Parse(time.RFC3339,map_mess["start_date"])
if err != nil{
logger.Logger.Error().Msg(err.Error())
}
fmt.Println("Catching " + str + " workflow... " + map_mess["id"])
if delete {
Bookings.DeleteSchedules(map_mess["id"])
} else {
s.getNextScheduledWorkflows(1)
stop, err := time.Parse(time.RFC3339,map_mess["stop_date"])
if err != nil{
logger.Logger.Error().Msg(err.Error())
}
s.bookings.AddSchedule(models.Booking{Workflow: map_mess["workflow"], Start: start, Stop: stop })
s.bookings.Mu.Unlock()
}
}
// At the moment very simplistic, but could be useful if we send bigger messages
func retrieveMapFromSub(message []byte) (result_map map[string]string) {
json.Unmarshal(message, &result_map)
return
}
// Used at launch of the component to retrieve the next scheduled workflows
// and then every X minutes in case some workflows were scheduled before launch
func (s *ScheduleManager) SchedulePolling() {
var sleep_time float64 = 1
for {
s.getNextScheduledWorkflows(1)
s.Logger.Info().Msg("Current list of schedules -------> " + fmt.Sprintf("%v", len(Bookings.Bookings)))
time.Sleep(time.Minute * time.Duration(sleep_time))
func (s *ScheduleManager) SchedulePolling (){
for(true){
err := s.getNextScheduledWorkflows(s.Api_url, 0.3)
if err != nil {
logger.Logger.Fatal().Msg("Failed to get the workspaces list, check api url and that api server is up : " + s.Api_url)
}
logger.Logger.Info().Msg("Current list of schedules")
fmt.Println(s.bookings.Bookings)
time.Sleep(time.Minute * 5)
}
}
func (s *ScheduleManager) getExecution(from time.Time, to time.Time) (exec_list []*workflow_execution.WorkflowExecution, err error) {
fmt.Printf("Getting workflows execution from %s to %s \n", from.String(), to.String())
f := dbs.Filters{
And: map[string][]dbs.Filter{
"execution_date": {{Operator: dbs.GTE.String(), Value: primitive.NewDateTimeFromTime(from)}, {Operator: dbs.LTE.String(), Value: primitive.NewDateTimeFromTime(to)}},
"state": {{Operator: dbs.EQUAL.String(), Value: 1}},
},
}
res := oclib.Search(&f, "", oclib.LibDataEnum(oclib.WORKFLOW_EXECUTION))
if res.Code != 200 {
s.Logger.Error().Msg("Error loading")
return
}
for _, exec := range res.Data {
exec_list = append(exec_list, exec.(*workflow_execution.WorkflowExecution))
}
return
}
func (s *ScheduleManager) getNextScheduledWorkflows(minutes float64) {
func (s *ScheduleManager) getNextScheduledWorkflows(apiurl string, hours float64) (error) {
s.ws.Init(apiurl)
params := url.Values{}
start := time.Now().UTC()
if next_wf_exec, err := s.getExecution(
start.Add(time.Second*time.Duration(-1)).UTC(),
start.Add(time.Minute*time.Duration(minutes)).UTC(),
); err != nil {
s.Logger.Error().Msg("Could not retrieve next schedules")
} else {
Bookings.AddSchedules(next_wf_exec, s.Logger)
params.Add("start_date", start.Format(time.RFC3339))
time_span := time.Hour * time.Duration(hours)
params.Add("stop_date",start.Add(time_span).Format(time.RFC3339))
body, err := s.ws.Get("v1/schedule?" + params.Encode())
if err != nil {
return err
}
var workflows []map[string]string
json.Unmarshal(body,&workflows)
s.bookings.Mu.Lock()
defer s.bookings.Mu.Unlock()
for _, workflow := range(workflows){
start, _ := time.Parse(time.RFC3339,workflow["start_date"])
stop, _ := time.Parse(time.RFC3339,workflow["stop_date"])
s.bookings.AddSchedule(models.Booking{Workflow: workflow["Workflow"], Start: start, Stop: stop})
}
return nil
}

View File

@ -1,39 +0,0 @@
version: '3.4'
services:
nats:
image: 'nats:latest'
container_name: nats
ports:
- 4222:4222
command:
- "--debug"
networks:
- scheduler
- catalog
loki:
image: 'grafana/loki'
container_name: loki
ports :
- "3100:3100"
networks:
- scheduler
grafana:
image: 'grafana/grafana'
container_name: grafana
ports:
- '3000:3000'
networks:
- scheduler
volumes:
- ./conf/grafana_data_source.yml:/etc/grafana/provisioning/datasources/datasource.yml
#environment:
#- GF_SECURITY_ADMIN_PASSWORD=pfnirt # Change this to anything but admin to not have a password change page at startup
#- GF_SECURITY_ADMIN_USER=admin
#- GF_SECURITY_DISABLE_INITIAL_ADMIN_PASSWORD_CHANGE=true
networks:
scheduler:
external: true
catalog:
external: true

View File

@ -1,16 +1,19 @@
version: '3.4'
services:
oc-schedulerd:
environment:
- MONGO_DATABASE=DC_myDC
image: 'oc-schedulerd:latest'
nats:
image: 'nats:latest'
container_name: nats
ports:
- 9001:8080
container_name: oc-schedulerd
- 4222:4222
command:
- "--debug"
networks:
- catalog
networks:
catalog:
external: true
- scheduler
loki:
image: 'grafana/loki'
container_name: loki
ports :
- "3100:3100"
networks:
- scheduler

View File

@ -3,7 +3,7 @@ package "main" {
class Graph {
[]DataModel Datas
[]ComputingModel Computings
[]ComputeModel Computes
[]DatacenterModel Datacenters
[]StorageModel Storages
map[string, Link] Links
HttpQuery ws
@ -13,7 +13,7 @@ package "main" {
GetWorkflowComponents(workflow string)
GetLinks(workflow string)
AddDataModel(id string, user_input gjson.Result, wf_id string) error
AddComputeModel(id string, user_input gjson.Result, wf_id string) error
AddDatacenterModel(id string, user_input gjson.Result, wf_id string) error
AddComputingModel(id string, user_input gjson.Result, wf_id string) error
AddStorageModel(id string, user_input gjson.Result, wf_id string) error
ExportToArgo(id string) error

View File

@ -4,14 +4,14 @@ package "main" {
[]Link Links
[]DataModel Datas
[]ComputingModel Computings
[]ComputeModel Computes
[]DatacenterModel Datacenters
[]StorageModel Storages
HttpQuery ws
GetGraphList(apiurl string) (map[string]string, error)
LoadFrom(workspace string) error
AddDataModel(id string) error
AddComputeModel(id string) error
AddDatacenterModel(id string) error
AddComputingModel(id string) error
AddStorageModel(id string) error
ExportToArgo(id string) error

14
execution_manager_test.go Normal file
View File

@ -0,0 +1,14 @@
package main
import (
"oc-scheduler/daemons"
"oc-scheduler/models"
"testing"
)
func TestCreateManifest(t *testing.T){
em := daemons.ExecutionManager{}
em.CreateManifest(models.Booking{},"fessity-chlics_23_07_2024_154326")
}

43
go.mod
View File

@ -1,14 +1,15 @@
module oc-schedulerd
module oc-scheduler
go 1.22.0
toolchain go1.22.5
require (
cloud.o-forge.io/core/oc-catalog v0.0.0-20240704140805-4b03211502fb
github.com/beego/beego v1.12.12
github.com/beego/beego/v2 v2.3.1
github.com/beego/beego/v2 v2.2.2
github.com/goraz/onion v0.1.3
github.com/nats-io/nats.go v1.37.0
github.com/nats-io/nats.go v1.9.1
github.com/nwtgck/go-fakelish v0.1.3
github.com/rs/zerolog v1.33.0
github.com/tidwall/gjson v1.17.1
@ -17,7 +18,6 @@ require (
)
require (
cloud.o-forge.io/core/oc-lib v0.0.0-20241107122526-f3df1e42b9ba // indirect
github.com/Klathmon/StructToMap v0.0.0-20140724123129-3d0229e2dce7 // indirect
github.com/antihax/optional v1.0.0 // indirect
github.com/aws/aws-sdk-go v1.36.29 // indirect
@ -25,18 +25,14 @@ require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.6 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.22.1 // indirect
github.com/go-stack/stack v1.8.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/golang/snappy v0.0.2 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/uuid v1.6.0 // indirect
@ -45,48 +41,41 @@ require (
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.17.11 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/klauspost/compress v1.17.2 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/montanaflynn/stats v0.7.1 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nats-io/jwt v0.3.2 // indirect
github.com/nats-io/nkeys v0.4.7 // indirect
github.com/nats-io/nkeys v0.1.3 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_golang v1.19.0 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.48.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/shiena/ansicolor v0.0.0-20200904210342-c7312218db18 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/ugorji/go/codec v1.1.7 // indirect
github.com/vk496/cron v1.2.0 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/scram v1.1.2 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect
github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc // indirect
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
go.mongodb.org/mongo-driver v1.17.1 // indirect
golang.org/x/crypto v0.28.0 // indirect
golang.org/x/net v0.30.0 // indirect
go.mongodb.org/mongo-driver v1.4.5 // indirect
golang.org/x/crypto v0.23.0 // indirect
golang.org/x/net v0.23.0 // indirect
golang.org/x/oauth2 v0.16.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.26.0 // indirect
golang.org/x/term v0.25.0 // indirect
golang.org/x/text v0.19.0 // indirect
golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.20.0 // indirect
golang.org/x/term v0.20.0 // indirect
golang.org/x/text v0.15.0 // indirect
golang.org/x/time v0.3.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.35.1 // indirect
google.golang.org/protobuf v1.34.1 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
k8s.io/api v0.30.3 // indirect

140
go.sum
View File

@ -1,29 +1,7 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.o-forge.io/core/oc-lib v0.0.0-20240808075405-f45ad91687c4 h1:3xqz2s6r/PONqLKjoFX3P4OBYTn8eAfQNOT+zRVRCTE=
cloud.o-forge.io/core/oc-lib v0.0.0-20240808075405-f45ad91687c4/go.mod h1:V5EL+NV2s9P1/BcFm3/icfLeBYVVMLl1Z0F0eecJZGo=
cloud.o-forge.io/core/oc-lib v0.0.0-20240812075555-6e3069068ce4 h1:fdxRsT4eR4v1DM3FpTPi9AKxB5oIw3XgLu9ByNipj4I=
cloud.o-forge.io/core/oc-lib v0.0.0-20240812075555-6e3069068ce4/go.mod h1:V5EL+NV2s9P1/BcFm3/icfLeBYVVMLl1Z0F0eecJZGo=
cloud.o-forge.io/core/oc-lib v0.0.0-20240821093044-f64563c9ff06 h1:sYveE1C/0mpSr+ZmOYxuZ3fTWID7mr5hPiq0jQenv3Q=
cloud.o-forge.io/core/oc-lib v0.0.0-20240821093044-f64563c9ff06/go.mod h1:1hhYh5QWAbYw9cKplQ0ZD9PMgU8t6gPqiYF8sldv1HU=
cloud.o-forge.io/core/oc-lib v0.0.0-20240826085916-d0e1474f8f34 h1:40XQgwR9HxXSnouY+ZqE/xYCM4qa+U+RLA5GA5JSNyQ=
cloud.o-forge.io/core/oc-lib v0.0.0-20240826085916-d0e1474f8f34/go.mod h1:1hhYh5QWAbYw9cKplQ0ZD9PMgU8t6gPqiYF8sldv1HU=
cloud.o-forge.io/core/oc-lib v0.0.0-20240826094730-73dc43b329d7 h1:WrlURBiciau4p6iU3v0nKcQYjBqW8e9Uc2soDDXll28=
cloud.o-forge.io/core/oc-lib v0.0.0-20240826094730-73dc43b329d7/go.mod h1:1hhYh5QWAbYw9cKplQ0ZD9PMgU8t6gPqiYF8sldv1HU=
cloud.o-forge.io/core/oc-lib v0.0.0-20240826103423-aff9304b1a71 h1:GodGMXVFgSdd5R1FoUjFAloOS+zOd3j66Wa+jcEPa4c=
cloud.o-forge.io/core/oc-lib v0.0.0-20240826103423-aff9304b1a71/go.mod h1:1hhYh5QWAbYw9cKplQ0ZD9PMgU8t6gPqiYF8sldv1HU=
cloud.o-forge.io/core/oc-lib v0.0.0-20240926135235-3d58416d9ba7 h1:xVGDhiVCRvqV2fkXkpoFtUouriIheks4OtdxsaRlMOM=
cloud.o-forge.io/core/oc-lib v0.0.0-20240926135235-3d58416d9ba7/go.mod h1:FIJD0taWLJ5pjQLJ6sfE2KlTkvbmk5SMcyrxdjsaVz0=
cloud.o-forge.io/core/oc-lib v0.0.0-20240927065314-0ac55a0ec151 h1:Sjap/XMOz3ludAGHKk1GimzWKb2wFmoo5tdBQ1E8/Ro=
cloud.o-forge.io/core/oc-lib v0.0.0-20240927065314-0ac55a0ec151/go.mod h1:FIJD0taWLJ5pjQLJ6sfE2KlTkvbmk5SMcyrxdjsaVz0=
cloud.o-forge.io/core/oc-lib v0.0.0-20241002120813-a09a04e1a71e h1:77QHk5JSf0q13B/Ai3xjcsGSS7nX+9AfxcsYz5oDo/A=
cloud.o-forge.io/core/oc-lib v0.0.0-20241002120813-a09a04e1a71e/go.mod h1:t+zpCTVKVdHH/BImwtMYY2QIWLMXKgY4n/JhFm3Vpu8=
cloud.o-forge.io/core/oc-lib v0.0.0-20241015083538-9f5e6d60185a h1:2mBMc36WKh1/Dpomktx9dVXGxK0agFr7RdgvHTtyn2w=
cloud.o-forge.io/core/oc-lib v0.0.0-20241015083538-9f5e6d60185a/go.mod h1:t+zpCTVKVdHH/BImwtMYY2QIWLMXKgY4n/JhFm3Vpu8=
cloud.o-forge.io/core/oc-lib v0.0.0-20241107114600-4c0c75be9161 h1:so5V7C6kiJ9tpuxtgK/KcgjXQC2ythInAH8X2gohuaM=
cloud.o-forge.io/core/oc-lib v0.0.0-20241107114600-4c0c75be9161/go.mod h1:ya7Q+zHhaKM+XF6sAJ+avqHEVzaMnFJQih2X3TlTlGo=
cloud.o-forge.io/core/oc-lib v0.0.0-20241107122526-f3df1e42b9ba h1:MGd8N7bY1LWXMhAp7gibDNwMS2hsatLQ3rfayvy5rGs=
cloud.o-forge.io/core/oc-lib v0.0.0-20241107122526-f3df1e42b9ba/go.mod h1:ya7Q+zHhaKM+XF6sAJ+avqHEVzaMnFJQih2X3TlTlGo=
cloud.o-forge.io/core/oc-catalog v0.0.0-20240704140805-4b03211502fb h1:L5WH474LuiHZDkNWl5aH/+MRPeJ9RFlkBFEU32K3CAk=
cloud.o-forge.io/core/oc-catalog v0.0.0-20240704140805-4b03211502fb/go.mod h1:TyZCM2kCEEvz0uQW6gPQT0tJjyqFPGrbX8dHsrYVKKQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/Klathmon/StructToMap v0.0.0-20140724123129-3d0229e2dce7 h1:n0MD6UkwbgGHtXsmfgVzC2+ZbHzIsScpbq9ZGI18074=
github.com/Klathmon/StructToMap v0.0.0-20140724123129-3d0229e2dce7/go.mod h1:xdrQDwHlKUmv8yiElMx6W0W10cLkqpeSEUUib8KGtv4=
@ -60,9 +38,6 @@ github.com/beego/beego v1.12.12/go.mod h1:QURFL1HldOcCZAxnc1cZ7wrplsYR5dKPHFjmk6
github.com/beego/beego/v2 v2.0.1/go.mod h1:8zyHi1FnWO1mZLwTn62aKRIZF/aIKvkCBB2JYs+eqQI=
github.com/beego/beego/v2 v2.2.2 h1:h6TNybAiMPXx9RXxK71Wz+JkPE7rpsL+ctjSZpv5yB0=
github.com/beego/beego/v2 v2.2.2/go.mod h1:A3BC73uulBnqW3O1uBEN7q+oykprxipZTYRdZtEuKyY=
github.com/beego/beego/v2 v2.3.1 h1:7MUKMpJYzOXtCUsTEoXOxsDV/UcHw6CPbaWMlthVNsc=
github.com/beego/beego/v2 v2.3.1/go.mod h1:5cqHsOHJIxkq44tBpRvtDe59GuVRVv/9/tyVDxd5ce4=
github.com/beego/beego/v2 v2.3.2/go.mod h1:5cqHsOHJIxkq44tBpRvtDe59GuVRVv/9/tyVDxd5ce4=
github.com/beego/goyaml2 v0.0.0-20130207012346-5545475820dd/go.mod h1:1b+Y/CofkYwXMUU0OhQqGvsY2Bvgr4j6jfT699wyZKQ=
github.com/beego/x2j v0.0.0-20131220205130-a0352aadc542/go.mod h1:kSeGC/p1AbBiEp5kat81+DSQrZenVBZXklMLaELspWU=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@ -129,12 +104,6 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/gabriel-vasile/mimetype v1.4.4 h1:QjV6pZ7/XZ7ryI2KuyeEDE8wnh7fHP9YnQy+R0LnH8I=
github.com/gabriel-vasile/mimetype v1.4.4/go.mod h1:JwLei5XPtWdGiMFB5Pjle1oEeoSeEuJfJE+TtfvdB/s=
github.com/gabriel-vasile/mimetype v1.4.5 h1:J7wGKdGu33ocBOhGy0z653k/lFKLFDPJMG8Gql0kxn4=
github.com/gabriel-vasile/mimetype v1.4.5/go.mod h1:ibHel+/kbxn9x2407k1izTA1S81ku1z/DlgOW2QE0M4=
github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc=
github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/glendc/gopher-json v0.0.0-20170414221815-dc4743023d0c/go.mod h1:Gja1A+xZ9BoviGJNA2E9vFkPjjsl+CoJxSXiQM1UXtw=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@ -153,16 +122,8 @@ github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/
github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
github.com/go-playground/validator/v10 v10.22.0 h1:k6HsTZ0sTnROkhS//R0O+55JgM8C4Bx7ia+JlgcnOao=
github.com/go-playground/validator/v10 v10.22.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA=
github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
github.com/go-redis/redis v6.14.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
github.com/go-redis/redis/v7 v7.4.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
@ -225,8 +186,6 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8l
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw=
github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@ -321,12 +280,6 @@ github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0
github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/klauspost/compress v1.17.10 h1:oXAz+Vh0PMUvJczoi+flxpnBEPxoER1IaAnU/NMPtT0=
github.com/klauspost/compress v1.17.10/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@ -342,8 +295,6 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/ledisdb/ledisdb v0.0.0-20200510135210-d35789ec47e6/go.mod h1:n931TsDuKuq+uX4v1fulaMbA/7ZLLhjc85h7chZGBCQ=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
@ -362,8 +313,6 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
@ -388,8 +337,6 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@ -401,13 +348,9 @@ github.com/nats-io/nats-server/v2 v2.1.2 h1:i2Ly0B+1+rzNZHHWtD4ZwKi+OU5l+uQo1iDH
github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k=
github.com/nats-io/nats.go v1.9.1 h1:ik3HbLhZ0YABLto7iX80pZLPw/6dx3T+++MZJwLnMrQ=
github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w=
github.com/nats-io/nats.go v1.37.0 h1:07rauXbVnnJvv1gfIyghFEo6lUcYRY0WXc3x7x0vUxE=
github.com/nats-io/nats.go v1.37.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
github.com/nats-io/nkeys v0.1.3 h1:6JrEfig+HzTH85yxzhSVbjHRJv9cn0p6n3IngIcM5/k=
github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
@ -493,8 +436,6 @@ github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@ -577,39 +518,20 @@ github.com/vk496/cron v1.2.0 h1:fDxb4qNi6Rmxh3h9snW1sKJ0nHgjpg3fYc0Oq+igbvk=
github.com/vk496/cron v1.2.0/go.mod h1:f8lpm+SIXbjvujp8Dix4S2B+GGva/q0yrRPQ8hwTtOc=
github.com/wendal/errors v0.0.0-20130201093226-f66c77a7882b/go.mod h1:Q12BUT7DqIlHRmgv3RskH+UCM/4eqVMgI0EMmlSpAXc=
github.com/wendal/errors v0.0.0-20181209125328-7f31f4b264ec/go.mod h1:Q12BUT7DqIlHRmgv3RskH+UCM/4eqVMgI0EMmlSpAXc=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc h1:n+nNi93yXLkJvKwXNP9d55HC7lGK4H/SRcwB5IaUZLo=
github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/youmark/pkcs8 v0.0.0-20240424034433-3c2c7870ae76 h1:tBiBTKHnIjovYoLX/TPkcf+OjqqKGQrPtGT3Foz+Pgo=
github.com/youmark/pkcs8 v0.0.0-20240424034433-3c2c7870ae76/go.mod h1:SQliXeA7Dhkt//vS29v3zpbEwoa+zb2Cn5xj5uO4K5U=
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yuin/gopher-lua v0.0.0-20171031051903-609c9cd26973/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
go.etcd.io/etcd v3.3.25+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
go.mongodb.org/mongo-driver v1.4.5 h1:TLtO+iD8krabXxvY1F1qpBOHgOxhLWR7XsT7kQeRmMY=
go.mongodb.org/mongo-driver v1.4.5/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc=
go.mongodb.org/mongo-driver v1.16.0 h1:tpRsfBJMROVHKpdGyc1BBEzzjDUWjItxbVSZ8Ls4BQ4=
go.mongodb.org/mongo-driver v1.16.0/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw=
go.mongodb.org/mongo-driver v1.16.1 h1:rIVLL3q0IHM39dvE+z2ulZLp9ENZKThVfuvN/IiN4l8=
go.mongodb.org/mongo-driver v1.16.1/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw=
go.mongodb.org/mongo-driver v1.17.0 h1:Hp4q2MCjvY19ViwimTs00wHi7G4yzxh4/2+nTx8r40k=
go.mongodb.org/mongo-driver v1.17.0/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4=
go.mongodb.org/mongo-driver v1.17.1 h1:Wic5cJIwJgSpBhe3lx3+/RybR5PiYRMpVFgO7cOHyIM=
go.mongodb.org/mongo-driver v1.17.1/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4=
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@ -636,17 +558,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@ -659,7 +572,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -683,18 +595,8 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -709,11 +611,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -746,51 +645,21 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk=
golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU=
golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk=
golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM=
golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8=
golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24=
golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
@ -821,7 +690,6 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY
golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ=
golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -862,10 +730,6 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA=
google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

35
graph.xml Normal file

File diff suppressed because one or more lines are too long

15
logger/logger.go Normal file
View File

@ -0,0 +1,15 @@
package logger
import (
"os"
"time"
"github.com/rs/zerolog"
)
var Logger zerolog.Logger
func init() {
output := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC3339}
Logger = zerolog.New(output).With().Timestamp().Logger()
}

60
main.go
View File

@ -2,29 +2,59 @@ package main
import (
"fmt"
"os"
"oc-schedulerd/conf"
"oc-schedulerd/daemons"
conf "oc-scheduler/conf"
"oc-scheduler/models"
oclib "cloud.o-forge.io/core/oc-lib"
"oc-scheduler/daemons"
"oc-scheduler/logger"
)
// var log zerolog.Logger
func main() {
oclib.InitDaemon("oc-schedulerd")
oclib.SetConfig(
conf.GetConfig().MongoUrl,
conf.GetConfig().DBName,
conf.GetConfig().NatsUrl,
conf.GetConfig().LokiUrl,
conf.GetConfig().Logs,
)
sch_mngr := daemons.ScheduleManager{Logger: oclib.GetLogger()}
// output := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC3339}
// log = zerolog.New(output).With().Timestamp().Logger()
app_conf := conf.GetConfig()
apiurl := app_conf.OcCatalogUrl
if _, err := os.Stat("./argo_workflows/"); os.IsNotExist(err) {
os.Mkdir("./argo_workflows/",0755)
logger.Logger.Info().Msg("Created argo_workflows/")
}
var bookings models.ScheduledBooking
sch_mngr := daemons.ScheduleManager{Api_url: apiurl}
sch_mngr.SetBookings(&bookings)
exe_mngr := daemons.ExecutionManager{}
exe_mngr.SetBookings(&bookings)
go sch_mngr.ListenNATS()
go sch_mngr.SchedulePolling()
go exe_mngr.RetrieveNextExecutions()
exe_mngr.RetrieveNextExecutions()
sch_mngr.ListenForWorkflowSubmissions()
// method in Schedule manager that checks the first Schedule object for its start date and exe
// var g Graph
// list, err := g.GetGraphList(apiurl)
// if err != nil {
// log.Fatal().Msg("Failed to get the workspaces list, check api url and that api server is up : " + apiurl)
// }
// println("Available workspaces :")
// for workspace, _ := range list {
// println(workspace)
// }
// g.LoadFrom(list["test-alpr"])
// g.ExportToArgo("test-alpr")
fmt.Print("stop")
fmt.Print("stop")
}

View File

@ -4,12 +4,12 @@ metadata:
name: test-monitor
spec:
containers:
- name: "oc-workflow-prous-skintris"
- name: "oc-monitor-quity-anetran"
image: docker.io/library/oc-monitor # Currently uses the local contenaird
imagePullPolicy: IfNotPresent # This should be removed once a registry has been set up
env:
- name: "OCMONITOR_ARGOFILE"
value: "prous-skintris_29_07_2024_164008.yml"
value: "quity-anetran_29_07_2024_144136.yml"
- name: "OCMONITOR_LOKIURL"
value: "info" # !!!! In dev this must be replaced with the address of one of your interface (wifi, ethernet..)
restartPolicy: OnFailure

53
models/http.go Normal file
View File

@ -0,0 +1,53 @@
package models
import (
"io"
"net/http"
"net/http/cookiejar"
"net/url"
)
type HttpQuery struct {
baseurl string
jar http.CookieJar
Cookies map[string]string
}
func (h *HttpQuery) Init(url string) {
h.baseurl = url
h.jar, _ = cookiejar.New(nil)
h.Cookies = make(map[string]string)
}
func (h *HttpQuery) Get(url string) ([]byte, error) {
client := &http.Client{Jar: h.jar}
resp, err := client.Get(h.baseurl + url)
if err != nil {
return nil, err
}
// store received cookies
for _, cookie := range h.jar.Cookies(resp.Request.URL) {
h.Cookies[cookie.Name] = cookie.Value
}
if err != nil {
return nil, err
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return body, nil
}
func (h *HttpQuery) Post(url string, data url.Values) (*http.Response, error) {
client := &http.Client{Jar: h.jar}
resp, err := client.PostForm(h.baseurl+url, data)
if err != nil {
return nil, err
}
// store received cookies
for _, cookie := range h.jar.Cookies(resp.Request.URL) {
h.Cookies[cookie.Name] = cookie.Value
}
return resp, err
}

71
models/schedule.go Normal file
View File

@ -0,0 +1,71 @@
package models
import (
"fmt"
"oc-scheduler/logger"
"sync"
"time"
)
// Is duration really important ?
type Booking struct {
Start time.Time
Stop time.Time
Duration uint
Workflow string
}
type ScheduledBooking struct {
Bookings []Booking
Mu sync.Mutex
}
func (s Booking) Equals(other Booking) bool {
return s.Workflow == other.Workflow && s.Start == other.Start && s.Stop == other.Stop
}
func (sb *ScheduledBooking) AddSchedule(new_booking Booking){
if(!sb.scheduleAlreadyExists(new_booking)){
sb.Bookings = append(sb.Bookings,new_booking)
logger.Logger.Info().Msg("Updated list schedules : \n " + sb.String())
} else {
// Debug condition : delete once this feature is ready to be implemented
logger.Logger.Debug().Msg("Workflow received not added")
logger.Logger.Debug().Msg("current schedule contains")
for _, booking := range(sb.Bookings){
logger.Logger.Debug().Msg(booking.String())
}
}
}
func (sb *ScheduledBooking) GetListNames()(list_names []string ){
for _, schedule := range(sb.Bookings){
list_names = append(list_names, schedule.Workflow)
}
return
}
func (sb *ScheduledBooking) scheduleAlreadyExists(new_booking Booking) bool {
for _, booking := range(sb.Bookings){
if booking.Equals(new_booking){
return true
}
}
return false
}
func (b *Booking) String() string {
return fmt.Sprintf("{workflow : %s , start_date : %s , stop_date : %s }", b.Workflow, b.Start.Format(time.RFC3339), b.Stop.Format(time.RFC3339))
}
func (sb *ScheduledBooking) String() string {
var str string
for _, booking := range(sb.Bookings){
str += fmt.Sprintf("%s\n", booking.String())
}
return str
}

40
models/template_models.go Normal file
View File

@ -0,0 +1,40 @@
package models
type Parameter struct {
Name string `yaml:"name,omitempty"`
Value string `yaml:"value,omitempty"`
}
type Container struct {
Image string `yaml:"image"`
Command []string `yaml:"command,omitempty,flow"`
Args []string `yaml:"args,omitempty,flow"`
VolumeMounts []VolumeMount `yaml:"volumeMounts,omitempty"`
}
type VolumeMount struct {
Name string `yaml:"name"`
MountPath string `yaml:"mountPath"`
}
type Task struct {
Name string `yaml:"name"`
Template string `yaml:"template"`
Dependencies []string `yaml:"dependencies,omitempty"`
Arguments struct {
Parameters []Parameter `yaml:"parameters,omitempty"`
} `yaml:"arguments,omitempty"`
}
type Dag struct {
Tasks []Task `yaml:"tasks,omitempty"`
}
type Template struct {
Name string `yaml:"name"`
Inputs struct {
Parameters []Parameter `yaml:"parameters"`
} `yaml:"inputs,omitempty"`
Container Container `yaml:"container,omitempty"`
Dag Dag `yaml:"dag,omitempty"`
}

19
models/volume_models.go Normal file
View File

@ -0,0 +1,19 @@
package models
type VolumeClaimTemplate struct {
Metadata struct {
Name string `yaml:"name"`
} `yaml:"metadata"`
Spec VolumeSpec `yaml:"spec"`
}
type VolumeSpec struct {
AccessModes []string `yaml:"accessModes,flow"`
Resources struct {
Requests struct {
Storage string `yaml:"storage"`
} `yaml:"requests"`
} `yaml:"resources"`
}

Binary file not shown.

View File

@ -0,0 +1,277 @@
// A class that translates the informations held in the graph object
// via its lists of components into an argo file, using the a list of
// link ID to build the dag
package workflow_builder
import (
"fmt"
"os"
"slices"
"strings"
"time"
. "oc-scheduler/models"
"github.com/beego/beego/v2/core/logs"
"github.com/nwtgck/go-fakelish"
"gopkg.in/yaml.v3"
)
type ArgoBuilder struct {
graph Graph
branches [][]string
Workflow Workflow
}
type Workflow struct {
ApiVersion string `yaml:"apiVersion"`
Kind string `yaml:"kind"`
Metadata struct {
GenerateName string `yaml:"generateName"`
} `yaml:"metadata"`
Spec Spec `yaml:"spec,omitempty"`
}
type Spec struct {
Entrypoint string `yaml:"entrypoint"`
Arguments []Parameter `yaml:"arguments,omitempty"`
Volumes []VolumeClaimTemplate `yaml:"volumeClaimTemplates,omitempty"`
Templates []Template `yaml:"templates"`
}
func (b *ArgoBuilder) CreateDAG() (string, error) {
fmt.Println("list of branches : ", b.branches)
b.createTemplates()
b.createDAGstep()
b.createVolumes()
b.Workflow.Spec.Entrypoint = "dag"
b.Workflow.ApiVersion = "argoproj.io/v1alpha1"
b.Workflow.Kind = "Workflow"
random_name := generateWfName()
b.Workflow.Metadata.GenerateName = "oc-test-" + random_name
yamlified, err := yaml.Marshal(b.Workflow)
if err != nil {
logs.Error("Could not transform object to yaml file")
return "", err
}
// Give a unique name to each argo file with its timestamp DD:MM:YYYY_hhmmss
current_timestamp := time.Now().Format("02_01_2006_150405")
file_name := random_name + "_" + current_timestamp + ".yml"
workflows_dir := "argo_workflows/"
err = os.WriteFile(workflows_dir + file_name , []byte(yamlified), 0660)
if err != nil {
logs.Error("Could not write the yaml file")
return "",err
}
return file_name, nil
}
func (b *ArgoBuilder) createTemplates() {
for _, comp := range b.graph.Computings{
image_name := strings.Split(comp.Command," ")[0] // TODO : decide where to store the image name, GUI or models.computing.Image
temp_container := Container{Image: image_name} // TODO : decide where to store the image name, GUI or models.computing.Image
temp_container.Command = getComputingCommands(comp.Command)
temp_container.Args = getComputingArgs(comp.Arguments,comp.Command)
input_names := getComputingEnvironmentName(comp.Environment)
var inputs_container []Parameter
for _, name := range input_names {
inputs_container = append(inputs_container, Parameter{Name: name})
}
argo_name := getArgoName(comp.Name,comp.ID)
new_temp := Template{Name: argo_name, Container: temp_container}
new_temp.Inputs.Parameters = inputs_container
new_temp.Container.VolumeMounts = append(new_temp.Container.VolumeMounts, VolumeMount{Name: "workdir",MountPath: "/mnt/vol"}) // TODO : replace this with a search of the storage / data source name
b.Workflow.Spec.Templates = append(b.Workflow.Spec.Templates, new_temp)
}
}
func (b *ArgoBuilder) createDAGstep() {
new_dag := Dag{}
for _, comp := range b.graph.Computings{
unique_name := getArgoName(comp.Name,comp.ID)
step := Task{Name: unique_name, Template: unique_name}
comp_envs := getComputingEnvironment(comp.Environment)
for name, value := range comp_envs {
step.Arguments.Parameters = append(step.Arguments.Parameters, Parameter{Name: name, Value: value})
}
// retrieves the name (computing.name-computing.ID)
step.Dependencies = b.getDependency(comp.ID)
new_dag.Tasks = append(new_dag.Tasks, step)
}
b.Workflow.Spec.Templates = append (b.Workflow.Spec.Templates, Template{Name: "dag", Dag: new_dag})
}
func (b *ArgoBuilder) createVolumes() {
// For testing purposes we only declare one volume, mounted in each computing
new_volume := VolumeClaimTemplate{}
new_volume.Metadata.Name = "workdir"
new_volume.Spec.AccessModes = []string{"ReadWriteOnce"}
new_volume.Spec.Resources.Requests.Storage = "1Gi"
b.Workflow.Spec.Volumes = append(b.Workflow.Spec.Volumes, new_volume)
}
func (b *ArgoBuilder) getDependency(current_computing_id string) (dependencies []string) {
var dependencies_id []string
for _, link := range b.graph.Links {
if current_computing_id == link.Destination && b.graph.getComponentType(link.Source) == "computing" && !slices.Contains(dependencies_id,link.Source) {
dependencies_id = append(dependencies_id, link.Source)
}
}
for _, dependency := range dependencies_id {
dependency_name := getArgoName(b.graph.getComponentName(dependency),dependency)
dependencies = append(dependencies, dependency_name)
}
return
}
// func (b *ArgoBuilder) componentInBranch(component_id string, branch []string) bool {
// for _, link := range branch {
// if b.graph.Links[link].Source == component_id || b.graph.Links[link].Destination == component_id {
// return true
// }
// }
// return false
// }
// func (b *ArgoBuilder) findPreviousComputing(computing_id string, branch []string, index int) string {
// for i := index; i >= 0 ; i-- {
// previousLink := b.graph.Links[branch[i]]
// if previousLink.Source != computing_id && b.graph.getComponentType(previousLink.Source) == "computing"{
// name := getArgoName(b.graph.getComponentName(previousLink.Source),previousLink.Source)
// return name
// }
// if previousLink.Destination != computing_id && b.graph.getComponentType(previousLink.Destination) == "computing"{
// name := getArgoName(b.graph.getComponentName(previousLink.Destination),previousLink.Destination)
// return name
// }
// }
// return ""
// }
func getComputingCommands(user_input string) (list_command []string) {
user_input = removeImageName(user_input)
if len(user_input) == 0 {
return
}
list_command = strings.Split(user_input, " ")
for i := range list_command {
list_command[i] = list_command[i]
}
return
}
func getComputingArgs(user_input []string, command string) (list_args []string) {
if len(user_input) == 0 {
return
}
// quickfix that might need improvement
if(strings.Contains(command,"sh -c")){
list_args = append(list_args, strings.Join(user_input," "))
return
}
for _, arg := range user_input{
list_args = append(list_args, arg)
}
return
}
// Currently implements code to overcome problems in data structure
func getComputingEnvironment(user_input []string) (map_env map[string]string) {
if len(user_input) == 0 {
return
}
if(len(user_input) == 1){
user_input = strings.Split(user_input[0],",")
}
map_env = make(map[string]string,0)
for _, str := range user_input {
new_pair := strings.Split(str,"=")
if(len(new_pair) != 2) {
logs.Error("Error extracting the environment variable from ", str)
panic(0)
}
map_env[new_pair[0]] = new_pair[1]
}
return
}
func getComputingEnvironmentName(user_input []string) (list_names []string){
env_map := getComputingEnvironment(user_input)
for name := range env_map {
list_names = append(list_names, name)
}
return
}
func generateWfName() (Name string){
Name = fakelish.GenerateFakeWord(5, 8) + "-" + fakelish.GenerateFakeWord(5, 8)
return
}
func getArgoName(raw_name string, component_id string) (formatedName string){
formatedName = strings.ReplaceAll(raw_name," ","-")
formatedName += "-" + component_id
formatedName = strings.ToLower(formatedName)
return
}
func printYAML(data interface{}) {
yamlData, err := yaml.Marshal(data)
if err != nil {
fmt.Printf("Error marshalling YAML: %v\n", err)
return
}
fmt.Println(string(yamlData))
}
func removeImageName(user_input string) string {
// First command is the name of the container for now
if len(strings.Split(user_input, " ")) == 1 {
return ""
}
slice_input := strings.Split(user_input, " ")
new_slice := slice_input[1:]
user_input = strings.Join(new_slice," ")
return user_input
}

358
workflow_builder/graph.go Normal file
View File

@ -0,0 +1,358 @@
package workflow_builder
import (
"encoding/json"
"fmt"
"maps"
"oc-scheduler/conf"
"oc-scheduler/logger"
models "oc-scheduler/models"
catalog_models "cloud.o-forge.io/core/oc-catalog/models" // this will be replaced with oc-lib
"github.com/beego/beego/v2/core/logs"
"github.com/tidwall/gjson"
)
type Graph struct {
workflow_name string // used to test if the graph has been instatiated, private so can only be set by a graph's method
Datas []catalog_models.DataModel
Computings []catalog_models.ComputingModel
Datacenters []catalog_models.DatacenterModel
Storages []catalog_models.StorageModel
Links map[string]catalog_models.Link
ws models.HttpQuery
}
// Create a dictionnaries with each existing workflow from a workspace, associated to the JSON representation of its content
func (g *Graph) GetGraphList(apiurl string) (map[string]string, error) {
g.ws.Init(apiurl)
body, err := g.ws.Get("v1/workspace/list")
if err != nil {
return nil, err
}
workspaces := make(map[string]string)
result := gjson.Get(string(body), "Workflows")
result.ForEach(func(key, value gjson.Result) bool {
workspaces[key.Str] = value.String()
return true // keep iterating
})
return workspaces, nil
}
// Should the parameter be removed, since we have oc-catalog url in the conf ?
func (g *Graph) GetGraph(apiurl string, workflow string) (string, error) {
g.ws.Init(apiurl)
body, err := g.ws.Get("v1/workflow/" + workflow)
if err != nil {
return "", err
}
graph := string(body)
// result := gjson.Get(string(body), "Workflows")
// result.ForEach(func(key, value gjson.Result) bool {
// workspaces[key.Str] = value.String()
// return true // keep iterating
// })
return graph, nil
}
// Create the objects from the mxgraphxml stored in the workflow given as a parameter
func (g *Graph) LoadFrom(workflow_name string) error {
// Extract the xmlgraph from the given workspace
graph, err := g.GetGraph(conf.GetConfig().OcCatalogUrl,workflow_name)
if err != nil {
return err
}
// os.WriteFile("graph.xml", []byte(decodedValue), 0660)
g.GetWorkflowComponents(graph)
g.GetLinks(graph)
g.workflow_name = workflow_name
return nil
}
// Create the objects that correspond to each component
// in a workflow, combining the user input and the base components attributes
func (g *Graph) GetWorkflowComponents(workflow string){
types := []string{"computing","datacenter","data","storage"} // create a constant for more maintainability OR even better get the list of all component's type for this WF
for _, component_type := range types {
// Retrieve the dict of component for a specific type in the workflow
result := gjson.Get(workflow, component_type)
if (result.Type != gjson.Null) {
result.ForEach(func(id, value gjson.Result) bool{
comp_id := value.Get("referenceID").Str
if (comp_id != "") {
switch component_type {
case "computing":
g.AddComputingModel(comp_id, value, id.Str)
case "data":
g.AddDataModel(comp_id, value, id.Str)
case "datacenter":
g.AddDatacenterModel(comp_id, value, id.Str)
case "storage":
g.AddStorageModel(comp_id, value, id.Str)
default :
logs.Critical("Component type doesn't match a know type : " + component_type)
}
}
return true
})
}
}
}
func (g *Graph) GetLinks(workflow string){
g.Links = make(map[string]catalog_models.Link)
result := gjson.Get(workflow, "link")
if (result.Type != gjson.Null) {
result.ForEach(func(id, value gjson.Result) bool{
var l catalog_models.Link
json.Unmarshal([]byte(value.Raw),&l)
g.Links[id.Str] = l
return true
})
}
}
func (g *Graph) AddDataModel(id string, user_input gjson.Result, wf_id string) error {
var d catalog_models.DataModel
resp, err := g.ws.Get("v1/data/" + id)
if err != nil {
return err
}
json.Unmarshal(resp, &d)
json.Unmarshal([]byte(user_input.Raw),&d.DataNEWModel)
d.ID = wf_id
g.Datas = append(g.Datas, d)
return nil
}
func (g *Graph) AddDatacenterModel(id string, user_input gjson.Result, wf_id string) error {
var d catalog_models.DatacenterModel
resp, err := g.ws.Get("v1/datacenter/" + id)
if err != nil {
return err
}
json.Unmarshal(resp, &d)
json.Unmarshal([]byte(user_input.Raw),&d.DatacenterNEWModel)
d.ID = wf_id
g.Datacenters = append(g.Datacenters, d)
return nil
}
func (g *Graph) AddComputingModel(id string, user_input gjson.Result, wf_id string) error {
var c catalog_models.ComputingModel
resp, err := g.ws.Get("v1/computing/" + id)
if err != nil {
return err
}
json.Unmarshal(resp, &c)
json.Unmarshal([]byte(user_input.Raw),&c.ComputingNEWModel)
c.ID = wf_id
g.Computings = append(g.Computings, c)
return nil
}
func (g *Graph) AddStorageModel(id string, user_input gjson.Result, wf_id string) error {
var s catalog_models.StorageModel
resp, err := g.ws.Get("v1/storage/" + id)
if err != nil {
return err
}
json.Unmarshal(resp, &s)
json.Unmarshal([]byte(user_input.Raw),&s.StorageNEWModel)
s.ID = wf_id
g.Storages = append(g.Storages, s)
return nil
}
func (g *Graph) ExportToArgo() (string, error) {
if len(g.workflow_name) == 0 {
return "",fmt.Errorf("can't export a graph that has not been loaded yet")
}
end_links := make(map[string]catalog_models.Link)
for i, link := range g.Links {
if (!link.DCLink && !g.isSource(link.Destination,i)){
end_links[i] = link
}
}
// index_list := make([]int, len(g.Links))
// list_branches := make([][]string,0)
list_branches := g.getListBranches(end_links, nil,nil)
for _, branch := range list_branches{
str := ""
for _, link := range branch{
str = str + " --> " + g.getComponentName(g.Links[link].Source) + " linked with " + g.getComponentName(g.Links[link].Destination)
}
fmt.Println(str)
}
fmt.Println("Identified branches : ", list_branches)
argo_builder := ArgoBuilder{graph : *g, branches: list_branches}
filename, err := argo_builder.CreateDAG()
if err != nil {
logger.Logger.Error().Msg("Could not create the argo file for " + g.workflow_name)
return "", err
}
return filename, nil
}
// Return a list containing the IDs of each link that make up a branch in the graph
func (g *Graph) getListBranches(end_links map[string]catalog_models.Link, unvisited_links_list map[string]catalog_models.Link, current_branch []string) (list_branches [][]string) {
if current_branch == nil {
current_branch = make([]string, 0)
}
if unvisited_links_list == nil {
unvisited_links_list = make(map[string]catalog_models.Link,len(g.Links))
maps.Copy(unvisited_links_list,g.Links)
fmt.Println(unvisited_links_list)
}
for link_id, _ := range end_links {
j := link_id
new_branches := make([][]string,0)
previous_index := g.getPreviousLink(j, unvisited_links_list)
if len(previous_index) == 0 {
list_branches = append(list_branches, []string{link_id})
}
for _, id_link := range previous_index {
current_branch = append([]string{link_id},current_branch...)
delete(unvisited_links_list, link_id)
// create a new branch for each previous link, appending the current path to this node to the created branch
new_end_link := make(map[string]catalog_models.Link,0)
new_end_link[id_link] = g.Links[id_link]
new_branches = g.getListBranches(new_end_link,unvisited_links_list,current_branch)
for _, new_branch := range new_branches{
current_branch = append(new_branch,link_id)
list_branches = append(list_branches, current_branch)
}
}
}
return
}
func (g *Graph) ExportToHelm(id string) error {
return nil
}
// Return if it exists a link where Destination is the same as comp_id
func (g *Graph) isDestination(comp_id string,link_id string) bool {
for i, link := range g.Links{
if(i !=link_id && link.Destination == comp_id){
return true
}
}
return false
}
// Return if it exists a link where Source is the same as comp_id
func (g *Graph) isSource(comp_id string,link_id string) bool {
for i, link := range g.Links{
if(i !=link_id && link.Source == comp_id && !link.DCLink){
return true
}
}
return false
}
// Returns an index number if their is a link in g.Links
// with the same Destination id that the Source id in g.Links[linkIndex]
// or nil if not
func (g *Graph) getPreviousLink(link_id string,map_link map[string]catalog_models.Link) (previous_id []string) {
for k, link := range map_link{
if(k != link_id && link.Destination == g.Links[link_id].Source){
previous_id = append(previous_id, k)
}
}
return
}
func (g *Graph) getComponentName(id string) string {
for _, comp := range g.Computings{
if comp.ID == id {
return comp.Name
}
}
for _, storage := range g.Storages{
if storage.ID == id {
return storage.Name
}
}
for _, data := range g.Datas{
if data.ID == id {
return data.Name
}
}
return ""
}
// returns either computing, data or storage
func (g *Graph) getComponentType(component_id string) string {
for _, comp := range g.Computings {
if comp.ID == component_id{
return "computing"
}
}
for _, data := range g.Datas {
if data.ID == component_id{
return "data"
}
}
for _, storage := range g.Storages {
if storage.ID == component_id{
return "storage"
}
}
return ""
}
// Returns a slice of id, in case the link is made of twice the same type of component
func (g *Graph) getComponentByType(compType string, link catalog_models.Link) (ids []string){
if(g.getComponentType(link.Source) == compType){
ids = append(ids, link.Source)
}
if(g.getComponentType(link.Destination) == compType){
ids = append(ids, link.Destination)
}
return
}