110 lines
3.7 KiB
Go
110 lines
3.7 KiB
Go
package daemons
|
|
|
|
import (
|
|
"fmt"
|
|
"time"
|
|
|
|
"oc-schedulerd/conf"
|
|
|
|
oclib "cloud.o-forge.io/core/oc-lib"
|
|
"cloud.o-forge.io/core/oc-lib/dbs"
|
|
"cloud.o-forge.io/core/oc-lib/models/common/enum"
|
|
workflow_execution "cloud.o-forge.io/core/oc-lib/models/workflow_execution"
|
|
"go.mongodb.org/mongo-driver/bson/primitive"
|
|
)
|
|
|
|
var Executions = ScheduledExecution{Execs: map[string]workflow_execution.WorkflowExecution{}}
|
|
|
|
type ExecutionManager struct{}
|
|
|
|
// Loop every second on the Execution's list and move the Execution that must start to a new list
|
|
// that will be looped over to start them
|
|
func (em *ExecutionManager) RetrieveNextExecutions() {
|
|
logger := oclib.GetLogger()
|
|
for {
|
|
Executions.Mu.Lock()
|
|
if len(Executions.Execs) > 0 {
|
|
executions := Executions.Execs
|
|
orderedExec := map[int]map[string]workflow_execution.WorkflowExecution{}
|
|
for execId, exec := range executions {
|
|
if orderedExec[exec.Priority] == nil {
|
|
orderedExec[exec.Priority] = map[string]workflow_execution.WorkflowExecution{}
|
|
}
|
|
orderedExec[exec.Priority][execId] = exec
|
|
}
|
|
for i := range []int{7, 6, 5, 4, 3, 2, 1, 0} { // priority in reversed
|
|
if orderedExec[i] == nil {
|
|
continue
|
|
}
|
|
|
|
lead := time.Duration(conf.GetConfig().PrepLeadSeconds) * time.Second
|
|
for execId, exec := range orderedExec[i] {
|
|
if i == 0 && em.isAStartingExecutionBeforeEnd(&exec) { // BEST EFFORT exception
|
|
continue
|
|
}
|
|
// Fire PrepLeadSeconds before the scheduled start so oc-monitord
|
|
// has time to pre-pull images and set up infra before ExecDate.
|
|
if exec.ExecDate.Before(time.Now().UTC().Add(lead)) {
|
|
logger.Info().Msg(fmt.Sprintf("Launching prep for %s (scheduled %s, lead %s)",
|
|
execId, exec.ExecDate.Format(time.RFC3339), lead))
|
|
// Mark as STARTED immediately (before goroutine) so the next
|
|
// SchedulePolling cycle doesn't re-pick this execution from DB.
|
|
oclib.NewRequestAdmin(oclib.LibDataEnum(oclib.WORKFLOW_EXECUTION), nil).UpdateOne(map[string]interface{}{
|
|
"state": enum.STARTED.EnumIndex(),
|
|
}, exec.GetID())
|
|
go em.executeExecution(&exec)
|
|
delete(executions, execId)
|
|
}
|
|
}
|
|
}
|
|
|
|
}
|
|
Executions.Mu.Unlock()
|
|
time.Sleep(time.Second)
|
|
}
|
|
}
|
|
|
|
func (em *ExecutionManager) isAStartingExecutionBeforeEnd(execution *workflow_execution.WorkflowExecution) bool {
|
|
access := workflow_execution.NewAccessor(nil)
|
|
l, _, err := access.Search(&dbs.Filters{
|
|
And: map[string][]dbs.Filter{
|
|
"execution_date": {{Operator: dbs.LTE.String(), Value: primitive.NewDateTimeFromTime(*execution.EndDate)}},
|
|
"state": {{Operator: dbs.EQUAL.String(), Value: enum.SCHEDULED}},
|
|
}, // TODO later should refine on each endpoint
|
|
}, "", false)
|
|
if err != nil && len(l) == 0 {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func (em *ExecutionManager) executeExecution(execution *workflow_execution.WorkflowExecution) {
|
|
// start execution
|
|
// create the yaml that describes the pod : filename, path/url to Loki
|
|
var executor Executor
|
|
// exec_method := os.Getenv("MONITOR_METHOD")
|
|
logger := oclib.GetLogger()
|
|
|
|
duration := 0
|
|
if execution.EndDate != nil {
|
|
duration = int(execution.EndDate.Sub(execution.ExecDate).Seconds())
|
|
}
|
|
|
|
if conf.GetConfig().Mode == "kubernetes" {
|
|
executor = NewContainerMonitor(execution.UUID, execution.CreatorID, duration, execution.ExecDate)
|
|
} else {
|
|
executor = NewLocalMonitor(execution.UUID, execution.CreatorID, duration, execution.ExecDate)
|
|
}
|
|
|
|
if executor == nil {
|
|
logger.Fatal().Msg("Could not create executor")
|
|
oclib.NewRequestAdmin(oclib.LibDataEnum(oclib.WORKFLOW_EXECUTION), nil).UpdateOne(map[string]interface{}{
|
|
"state": enum.FAILURE.EnumIndex(),
|
|
}, execution.GetID())
|
|
return
|
|
}
|
|
|
|
args := executor.PrepareMonitorExec()
|
|
executor.LaunchMonitor(args, execution.GetID(), conf.GetConfig().KubeNamespace, logger)
|
|
}
|