core/oc-schedulerd
6
0
Fork 0
Code Issues 7 Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'feature/order' into main

Browse Source
This commit is contained in:
mr
2025-02-18 08:31:54 +01:00
parent 8ddb119899 b753523c35
commit 88610c3dba
15 changed files with 129 additions and 345 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
daemons/execute_monitor_kube.go
View File

@@ -1,67 +0,0 @@
package daemons
// type manifestValues struct{
// ARGO_FILE string
// LOKI_URL string
// CONTAINER_NAME string
// }
// manifest, err := em.CreateManifest(booking, argo_file_path)
// if err != nil {
// logger.Logger.Error().Msg("Could not create manifest " + err.Error())
// }
// // launch a pod that contains oc-monitor, give it the name of the workflow
// cmd := exec.Command("kubectl","apply","-f", "manifests/"+manifest)
// output , err := cmd.CombinedOutput()
// if err != nil {
// logger.Logger.Error().Msg("failed to create new pod for " + booking.Workflow + " :" + err.Error())
// return
// }
// func (*ExecutionManager) CreateManifest(booking models.Booking, argo_file string) (manifest_filepath string, err error) {
// var filled_template bytes.Buffer
// manifest_file, err := os.ReadFile("conf/monitor_pod_template.yml")
// if err != nil {
// logger.Logger.Error().Msg("Could not open the k8s template file for " + booking.Workflow)
// return "", err
// }
// container_name := getContainerName(argo_file)
// tmpl, err := template.New("manifest_template").Parse(string(manifest_file))
// if err != nil {
// logger.Logger.Error().Msg(err.Error())
// return "", err
// }
// manifest_data := manifestValues{
// ARGO_FILE: argo_file,
// LOKI_URL: conf.GetConfig().Logs,
// CONTAINER_NAME: container_name,
// }
// err = tmpl.Execute(&filled_template, manifest_data)
// if err != nil {
// logger.Logger.Error().Msg("Could not complete manifest template for " + booking.Workflow)
// return "", err }
// manifest_filepath = booking.Workflow + "_manifest.yaml"
// err = os.WriteFile("manifests/" + manifest_filepath, filled_template.Bytes(),0644)
// if err != nil {
// logger.Logger.Error().Msg("Could not write the YAML file for " + booking.Workflow + "'s manifest")
// return "", err
// }
// return manifest_filepath, nil
// }
// func getContainerName(argo_file string) string {
// regex := "([a-zA-Z]+-[a-zA-Z]+)"
// re := regexp.MustCompile(regex)
// container_name := re.FindString(argo_file)
// return container_name
// }

23
daemons/execute_monitor_local.go
View File

@@ -5,7 +5,6 @@ import (
"oc-schedulerd/conf"
"os/exec"
oclib "cloud.o-forge.io/core/oc-lib"
"github.com/rs/zerolog"
)
@@ -13,6 +12,7 @@ type LocalMonitor struct {
LokiURL string
KubeURL string
ExecutionID string
PeerID string
Duration int
Logger zerolog.Logger
}
@@ -21,17 +21,17 @@ func (lm *LocalMonitor) LaunchLocalMonitor() {
if lm.LokiURL == "" || lm.KubeURL == "" || lm.ExecutionID == "" {
lm.Logger.Error().Msg("Missing parameter in LocalMonitor")
}
// For dev purposes, in prod KubeURL must be a kube API's URL
if lm.KubeURL != "localhost" {
lm.execRemoteKube()
} else {
lm.execLocalKube()
}
lm.execKube()
}
func (lm *LocalMonitor) execLocalKube() {
args := []string{"-e", lm.ExecutionID, "-u", lm.LokiURL, "-m", oclib.GetConfig().MongoUrl, "-d", oclib.GetConfig().MongoDatabase}
func (lm *LocalMonitor) execKube() {
args := []string{"-e", lm.ExecutionID, "-p", lm.PeerID, "-u", lm.LokiURL, "-m", conf.GetConfig().MongoUrl,
"-d", conf.GetConfig().DBName}
if conf.GetConfig().Mode == "kubernetes" {
args = append(args, []string{"-M", conf.GetConfig().Mode, "-H", conf.GetConfig().KubeHost, "-P", conf.GetConfig().KubePort,
"-C", conf.GetConfig().KubeCert, "-D", conf.GetConfig().KubeData, "-c", conf.GetConfig().KubeCA}...)
}
if lm.Duration > 0 {
args = append(args, "-t", fmt.Sprintf("%d", lm.Duration))
}
@@ -42,6 +42,3 @@ func (lm *LocalMonitor) execLocalKube() {
lm.Logger.Error().Msg("Could not start oc-monitor for " + lm.ExecutionID + " : " + err.Error())
}
}
// TODO : implement this
func (lm *LocalMonitor) execRemoteKube() {}

37
daemons/execution_manager.go
View File

@@ -1,6 +1,8 @@
package daemons
import (
"fmt"
"oc-schedulerd/conf"
"os"
"time"
@@ -8,33 +10,33 @@ import (
workflow_execution "cloud.o-forge.io/core/oc-lib/models/workflow_execution"
)
var Bookings = ScheduledBooking{Bookings: []*workflow_execution.WorkflowExecution{}}
var Executions = ScheduledExecution{Execs: []*workflow_execution.WorkflowExecution{}}
type ExecutionManager struct{}
// Loop every second on the booking's list and move the booking that must start to a new list
// Loop every second on the Execution's list and move the Execution that must start to a new list
// that will be looped over to start them
func (em *ExecutionManager) RetrieveNextExecutions() {
logger := oclib.GetLogger()
for {
logger.Debug().Msg("New loop")
Bookings.Mu.Lock()
if len(Bookings.Bookings) > 0 {
bookings := Bookings.Bookings
for i := len(bookings) - 1; i >= 0; i-- {
if bookings[i].ExecDate.Before(time.Now().UTC()) {
logger.Info().Msg("Will execute " + bookings[i].UUID + " soon")
go em.executeBooking(bookings[i])
Bookings.Bookings = append(bookings[:i], bookings[i+1:]...)
fmt.Println("Checking for executions", len(Executions.Execs))
Executions.Mu.Lock()
if len(Executions.Execs) > 0 {
executions := Executions.Execs
for i := len(executions) - 1; i >= 0; i-- {
if executions[i].ExecDate.Before(time.Now().UTC()) {
logger.Info().Msg("Will execute " + executions[i].UUID + " soon")
go em.executeExecution(executions[i])
Executions.Execs = append(executions[:i], executions[i+1:]...)
}
}
}
Bookings.Mu.Unlock()
Executions.Mu.Unlock()
time.Sleep(time.Second)
}
}
func (em *ExecutionManager) executeBooking(booking *workflow_execution.WorkflowExecution) {
func (em *ExecutionManager) executeExecution(Execution *workflow_execution.WorkflowExecution) {
// start execution
// create the yaml that describes the pod : filename, path/url to Loki
exec_method := os.Getenv("MONITOR_METHOD")
@@ -44,15 +46,16 @@ func (em *ExecutionManager) executeBooking(booking *workflow_execution.WorkflowE
} else {
logger.Debug().Msg("Executing oc-monitor localy")
duration := 0
if booking.EndDate != nil && booking.ExecDate != nil {
duration = int(booking.EndDate.Sub(*booking.ExecDate).Seconds())
if Execution.EndDate != nil {
duration = int(Execution.EndDate.Sub(Execution.ExecDate).Seconds())
}
monitor := LocalMonitor{
Logger: logger,
Duration: duration,
LokiURL: oclib.GetConfig().LokiUrl,
LokiURL: conf.GetConfig().LokiUrl,
KubeURL: "localhost",
ExecutionID: booking.UUID,
ExecutionID: Execution.UUID,
PeerID: Execution.CreatorID,
}
monitor.LaunchLocalMonitor()
}

47
daemons/schedule_manager.go
View File

@@ -8,6 +8,7 @@ import (
oclib "cloud.o-forge.io/core/oc-lib"
"cloud.o-forge.io/core/oc-lib/dbs"
"cloud.o-forge.io/core/oc-lib/models/common/enum"
"cloud.o-forge.io/core/oc-lib/models/workflow_execution"
"cloud.o-forge.io/core/oc-lib/tools"
"github.com/nats-io/nats.go"
@@ -15,35 +16,36 @@ import (
"go.mongodb.org/mongo-driver/bson/primitive"
)
type ScheduledBooking struct {
Bookings []*workflow_execution.WorkflowExecution
Mu sync.Mutex
type ScheduledExecution struct {
Execs []*workflow_execution.WorkflowExecution
Mu sync.Mutex
}
func (sb *ScheduledBooking) DeleteSchedules(workflow_id string) {
func (sb *ScheduledExecution) DeleteSchedules(workflow_id string) {
toNotDelete := []*workflow_execution.WorkflowExecution{}
for _, b := range sb.Bookings {
for _, b := range sb.Execs {
if b.WorkflowID != workflow_id {
toNotDelete = append(toNotDelete, b)
}
}
Bookings.Mu.Lock()
defer Bookings.Mu.Unlock()
sb.Bookings = toNotDelete
Executions.Mu.Lock()
defer Executions.Mu.Unlock()
sb.Execs = toNotDelete
}
func (sb *ScheduledBooking) AddSchedules(new_bookings []*workflow_execution.WorkflowExecution, logger zerolog.Logger) {
Bookings.Mu.Lock()
defer Bookings.Mu.Unlock()
for _, exec := range new_bookings {
func (sb *ScheduledExecution) AddSchedules(new_executions []*workflow_execution.WorkflowExecution, logger zerolog.Logger) {
Executions.Mu.Lock()
defer Executions.Mu.Unlock()
for _, exec := range new_executions {
fmt.Println("Adding "+exec.UUID, !sb.execIsSet(exec))
if !sb.execIsSet(exec) {
sb.Bookings = append(sb.Bookings, exec)
sb.Execs = append(sb.Execs, exec)
}
}
}
func (sb *ScheduledBooking) execIsSet(exec *workflow_execution.WorkflowExecution) bool {
for _, b := range sb.Bookings {
func (sb *ScheduledExecution) execIsSet(exec *workflow_execution.WorkflowExecution) bool {
for _, b := range sb.Execs {
if b.Equals(exec) {
return true
}
@@ -99,7 +101,7 @@ func (s *ScheduleManager) listenForChange(nc *nats.Conn, chanName string, delete
}
fmt.Println("Catching " + str + " workflow... " + map_mess["id"])
if delete {
Bookings.DeleteSchedules(map_mess["id"])
Executions.DeleteSchedules(map_mess["id"])
} else {
s.getNextScheduledWorkflows(1)
}
@@ -112,7 +114,7 @@ func (s *ScheduleManager) SchedulePolling() {
var sleep_time float64 = 1
for {
s.getNextScheduledWorkflows(1)
s.Logger.Info().Msg("Current list of schedules -------> " + fmt.Sprintf("%v", len(Bookings.Bookings)))
s.Logger.Info().Msg("Current list of schedules -------> " + fmt.Sprintf("%v", len(Executions.Execs)))
time.Sleep(time.Minute * time.Duration(sleep_time))
}
}
@@ -121,10 +123,10 @@ func (s *ScheduleManager) getExecution(from time.Time, to time.Time) (exec_list
f := dbs.Filters{
And: map[string][]dbs.Filter{
"execution_date": {{Operator: dbs.GTE.String(), Value: primitive.NewDateTimeFromTime(from)}, {Operator: dbs.LTE.String(), Value: primitive.NewDateTimeFromTime(to)}},
"state": {{Operator: dbs.EQUAL.String(), Value: 1}},
"state": {{Operator: dbs.EQUAL.String(), Value: enum.SCHEDULED}},
},
}
res := oclib.Search(&f, "", oclib.LibDataEnum(oclib.WORKFLOW_EXECUTION))
res := oclib.NewRequest(oclib.LibDataEnum(oclib.WORKFLOW_EXECUTION), "", "", []string{}, nil).Search(&f, "", false)
if res.Code != 200 {
s.Logger.Error().Msg("Error loading")
return
@@ -132,17 +134,22 @@ func (s *ScheduleManager) getExecution(from time.Time, to time.Time) (exec_list
for _, exec := range res.Data {
exec_list = append(exec_list, exec.(*workflow_execution.WorkflowExecution))
}
fmt.Println("Found "+fmt.Sprintf("%v", len(exec_list))+" workflows", res)
return
}
func (s *ScheduleManager) getNextScheduledWorkflows(minutes float64) {
start := time.Now().UTC()
fmt.Println(s.getExecution(
start.Add(time.Second*time.Duration(-1)).UTC(),
start.Add(time.Minute*time.Duration(minutes)).UTC(),
))
if next_wf_exec, err := s.getExecution(
start.Add(time.Second*time.Duration(-1)).UTC(),
start.Add(time.Minute*time.Duration(minutes)).UTC(),
); err != nil {
s.Logger.Error().Msg("Could not retrieve next schedules")
} else {
Bookings.AddSchedules(next_wf_exec, s.Logger)
Executions.AddSchedules(next_wf_exec, s.Logger)
}
}