minimize code + schedulerd naming + docker
This commit is contained in:
parent
c710469881
commit
e5cfd6f4fb
18
.vscode/launch.json
vendored
Normal file
18
.vscode/launch.json
vendored
Normal file
@ -0,0 +1,18 @@
|
|||||||
|
{
|
||||||
|
// Use IntelliSense to learn about possible attributes.
|
||||||
|
// Hover to view descriptions of existing attributes.
|
||||||
|
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
|
||||||
|
"version": "0.2.0",
|
||||||
|
"configurations": [
|
||||||
|
{
|
||||||
|
"name": "Launch Package",
|
||||||
|
"type": "go",
|
||||||
|
"request": "launch",
|
||||||
|
"mode": "auto",
|
||||||
|
"program": "${fileDirname}",
|
||||||
|
"env": {
|
||||||
|
"MONITOR_METHOD" : "local"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
15
Dockerfile
15
Dockerfile
@ -4,15 +4,20 @@ ENV DOCKER_ENVIRONMENT=true
|
|||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
COPY conf/docker_scheduler.json /etc/oc/scheduler.json
|
|
||||||
|
|
||||||
RUN go build .
|
RUN go build .
|
||||||
|
|
||||||
FROM golang:alpine
|
FROM oc-monitord:latest AS monitord
|
||||||
|
|
||||||
|
FROM argoproj/argocd:latest
|
||||||
|
|
||||||
|
ENV MONITORD_PATH = "./oc-monitord"
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
COPY --from=builder /app/oc-scheduler .
|
COPY --from=monitord /app/oc-monitord .
|
||||||
COPY conf/docker_scheduler.json /etc/oc/scheduler.json
|
COPY --from=builder /app/oc-schedulerd .
|
||||||
|
COPY conf/docker_schedulerd.json /etc/oc/schedulerd.json
|
||||||
|
|
||||||
ENTRYPOINT ["/app/oc-scheduler"]
|
ENTRYPOINT ["/app/oc-schedulerd"]
|
||||||
|
|
||||||
|
33
conf/conf.go
33
conf/conf.go
@ -8,27 +8,26 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type Config struct {
|
type Config struct {
|
||||||
OcCatalogUrl string
|
MonitorPath string
|
||||||
MongoUrl string
|
OcCatalogUrl string
|
||||||
DBName string
|
MongoUrl string
|
||||||
Logs string
|
DBName string
|
||||||
LokiUrl string
|
Logs string
|
||||||
NatsUrl string
|
LokiUrl string
|
||||||
|
NatsUrl string
|
||||||
}
|
}
|
||||||
|
|
||||||
var instance *Config
|
var instance *Config
|
||||||
var once sync.Once
|
var once sync.Once
|
||||||
|
|
||||||
const defaultConfigFile = "/etc/oc/scheduler.json"
|
const defaultConfigFile = "/etc/oc/schedulerd.json"
|
||||||
const localConfigFile = "./conf/local_scheduler.json"
|
const localConfigFile = "./conf/local_schedulerd.json"
|
||||||
|
|
||||||
|
|
||||||
func init(){
|
|
||||||
|
|
||||||
|
func init() {
|
||||||
configFile := ""
|
configFile := ""
|
||||||
var o *onion.Onion
|
var o *onion.Onion
|
||||||
|
|
||||||
l3 := onion.NewEnvLayerPrefix("_", "OCSCHEDULER_")
|
l3 := onion.NewEnvLayerPrefix("_", "OCSCHEDULERD_")
|
||||||
l2, err := onion.NewFileLayer(defaultConfigFile, nil)
|
l2, err := onion.NewFileLayer(defaultConfigFile, nil)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
logs.Info("Config file found : " + defaultConfigFile)
|
logs.Info("Config file found : " + defaultConfigFile)
|
||||||
@ -49,13 +48,13 @@ func init(){
|
|||||||
} else if l2 == nil {
|
} else if l2 == nil {
|
||||||
o = onion.New(l1, l3)
|
o = onion.New(l1, l3)
|
||||||
}
|
}
|
||||||
|
GetConfig().MonitorPath = o.GetStringDefault("MONITORD_PATH", "../oc-monitord/oc-monitord")
|
||||||
GetConfig().OcCatalogUrl = o.GetStringDefault("oc-catalog", "https://localhost:49618")
|
GetConfig().OcCatalogUrl = o.GetStringDefault("oc-catalog", "https://localhost:49618")
|
||||||
GetConfig().Logs = o.GetStringDefault("loglevel", "info")
|
GetConfig().Logs = o.GetStringDefault("loglevel", "info")
|
||||||
GetConfig().LokiUrl = o.GetStringDefault("loki_url","http://127.0.0.1:3100")
|
GetConfig().LokiUrl = o.GetStringDefault("loki_url", "http://127.0.0.1:3100")
|
||||||
GetConfig().NatsUrl = o.GetStringDefault("nats_url","http://127.0.0.1:4222")
|
GetConfig().NatsUrl = o.GetStringDefault("nats_url", "http://127.0.0.1:4222")
|
||||||
GetConfig().MongoUrl = o.GetStringDefault("mongo_url","mongodb://127.0.0.1:27017")
|
GetConfig().MongoUrl = o.GetStringDefault("mongo_url", "mongodb://127.0.0.1:27017")
|
||||||
GetConfig().DBName = o.GetStringDefault("database_name","DC_myDC")
|
GetConfig().DBName = o.GetStringDefault("database_name", "DC_myDC")
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1,47 +1,41 @@
|
|||||||
package daemons
|
package daemons
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"oc-scheduler/conf"
|
"oc-schedulerd/conf"
|
||||||
"oc-scheduler/logger"
|
|
||||||
"os/exec"
|
"os/exec"
|
||||||
|
|
||||||
|
"github.com/rs/zerolog"
|
||||||
)
|
)
|
||||||
|
|
||||||
type LocalMonitor struct{
|
type LocalMonitor struct {
|
||||||
LokiURL string
|
LokiURL string
|
||||||
KubeURL string
|
KubeURL string
|
||||||
WorkflowName string
|
WorkflowName string
|
||||||
|
Logger zerolog.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
func (lm *LocalMonitor) LaunchLocalMonitor (){
|
func (lm *LocalMonitor) LaunchLocalMonitor() {
|
||||||
if (lm.LokiURL == "" || lm.KubeURL == "" || lm.WorkflowName == ""){
|
if lm.LokiURL == "" || lm.KubeURL == "" || lm.WorkflowName == "" {
|
||||||
logger.Logger.Error().Msg("Missing parameter in LocalMonitor")
|
lm.Logger.Error().Msg("Missing parameter in LocalMonitor")
|
||||||
}
|
}
|
||||||
|
|
||||||
// For dev purposes, in prod KubeURL must be a kube API's URL
|
// For dev purposes, in prod KubeURL must be a kube API's URL
|
||||||
if(lm.KubeURL == "localhost"){
|
if lm.KubeURL == "localhost" {
|
||||||
lm.execLocalKube()
|
lm.execLocalKube()
|
||||||
} else{
|
} else {
|
||||||
lm.execRemoteKube()
|
lm.execRemoteKube()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (lm *LocalMonitor) execLocalKube (){
|
func (lm *LocalMonitor) execLocalKube() {
|
||||||
// kube_url := ""
|
cmd := exec.Command(conf.GetConfig().MonitorPath, "-w", lm.WorkflowName, "-u", lm.LokiURL, "-m", conf.GetConfig().MongoUrl, "-d", conf.GetConfig().DBName)
|
||||||
cmd := exec.Command("../oc-monitor/oc-monitor", "-w",lm.WorkflowName, "-u", lm.LokiURL, "-m", conf.GetConfig().MongoUrl,"-d", conf.GetConfig().DBName)
|
|
||||||
// cmd_ls := exec.Command("ls", "../oc-monitor")
|
|
||||||
err := cmd.Start()
|
err := cmd.Start()
|
||||||
// output, err := cmd_ls.CombinedOutput()
|
if err != nil {
|
||||||
if err !=nil {
|
lm.Logger.Error().Msg("Could not start oc-monitor for " + lm.WorkflowName + " : " + err.Error())
|
||||||
logger.Logger.Error().Msg("Could not start oc-monitor for " + lm.WorkflowName + " : " + err.Error())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO : implement this
|
||||||
func (lm *LocalMonitor) execRemoteKube (){
|
func (lm *LocalMonitor) execRemoteKube() {
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func (lm *LocalMonitor) todo (){
|
|
||||||
|
|
||||||
}
|
}
|
@ -1,43 +1,30 @@
|
|||||||
package daemons
|
package daemons
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"oc-scheduler/conf"
|
"oc-schedulerd/conf"
|
||||||
"oc-scheduler/logger"
|
|
||||||
"oc-scheduler/models"
|
|
||||||
"os"
|
"os"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
oclib "cloud.o-forge.io/core/oc-lib"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ExecutionManager struct {
|
type ExecutionManager struct {
|
||||||
bookings *models.ScheduledBooking
|
bookings ScheduledBooking
|
||||||
executions []models.Booking
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
func (em *ExecutionManager) SetBookings(b *models.ScheduledBooking){
|
|
||||||
em.bookings = b
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Loop every second on the booking's list and move the booking that must start to a new list
|
// Loop every second on the booking's list and move the booking that must start to a new list
|
||||||
// that will be looped over to start them
|
// that will be looped over to start them
|
||||||
func (em *ExecutionManager) RetrieveNextExecutions(){
|
func (em *ExecutionManager) RetrieveNextExecutions() {
|
||||||
|
logger := oclib.GetLogger()
|
||||||
|
for {
|
||||||
if(em.bookings == nil){
|
logger.Debug().Msg("New loop")
|
||||||
logger.Logger.Error().Msg("booking has not been set in the exection manager")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
for(true){
|
|
||||||
logger.Logger.Debug().Msg("New loop")
|
|
||||||
em.bookings.Mu.Lock()
|
em.bookings.Mu.Lock()
|
||||||
bookings := em.bookings.Bookings
|
bookings := em.bookings.Bookings
|
||||||
if (len(bookings) > 0){
|
if len(bookings) > 0 {
|
||||||
for i := len( bookings) - 1 ; i >= 0 ; i--{
|
for i := len(bookings) - 1; i >= 0; i-- {
|
||||||
logger.Logger.Debug().Msg("It should start at " + bookings[i].Start.String() + " and it is now " + time.Now().UTC() .String())
|
logger.Debug().Msg("It should start at " + bookings[i].Start.String() + " and it is now " + time.Now().UTC().String())
|
||||||
if (bookings[i].Start.Before(time.Now().UTC())){
|
if bookings[i].Start.Before(time.Now().UTC()) {
|
||||||
logger.Logger.Info().Msg("Will execute " + bookings[i].Workflow + " soon")
|
logger.Info().Msg("Will execute " + bookings[i].Workflow + " soon")
|
||||||
go em.executeBooking(bookings[i])
|
go em.executeBooking(bookings[i])
|
||||||
bookings = append(bookings[:i], bookings[i+1:]...)
|
bookings = append(bookings[:i], bookings[i+1:]...)
|
||||||
em.bookings.Bookings = bookings
|
em.bookings.Bookings = bookings
|
||||||
@ -49,24 +36,19 @@ func (em *ExecutionManager) RetrieveNextExecutions(){
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (em *ExecutionManager) executeBooking(booking models.Booking){
|
func (em *ExecutionManager) executeBooking(booking Booking) {
|
||||||
|
|
||||||
|
|
||||||
// start execution
|
// start execution
|
||||||
// create the yaml that describes the pod : filename, path/url to Loki
|
// create the yaml that describes the pod : filename, path/url to Loki
|
||||||
|
|
||||||
|
|
||||||
exec_method := os.Getenv("MONITOR_METHOD")
|
exec_method := os.Getenv("MONITOR_METHOD")
|
||||||
|
logger := oclib.GetLogger()
|
||||||
if exec_method == "local"{
|
if exec_method == "k8s" {
|
||||||
logger.Logger.Debug().Msg("Executing oc-monitor localy")
|
logger.Error().Msg("TODO : executing oc-monitor in a k8s")
|
||||||
monitor := LocalMonitor{LokiURL: conf.GetConfig().LokiUrl,KubeURL: "localhost",WorkflowName: booking.Workflow,}
|
} else {
|
||||||
|
logger.Debug().Msg("Executing oc-monitor localy")
|
||||||
|
monitor := LocalMonitor{
|
||||||
|
Logger: logger,
|
||||||
|
LokiURL: conf.GetConfig().LokiUrl, KubeURL: "localhost",
|
||||||
|
WorkflowName: booking.Workflow}
|
||||||
monitor.LaunchLocalMonitor()
|
monitor.LaunchLocalMonitor()
|
||||||
}else{
|
|
||||||
logger.Logger.Error().Msg("TODO : executing oc-monitor in a k8s")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -3,45 +3,78 @@ package daemons
|
|||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"oc-scheduler/logger"
|
|
||||||
"oc-scheduler/models"
|
|
||||||
|
|
||||||
oclib "cloud.o-forge.io/core/oc-lib"
|
oclib "cloud.o-forge.io/core/oc-lib"
|
||||||
"cloud.o-forge.io/core/oc-lib/dbs"
|
"cloud.o-forge.io/core/oc-lib/dbs"
|
||||||
"cloud.o-forge.io/core/oc-lib/models/workflow_execution"
|
"cloud.o-forge.io/core/oc-lib/models/workflow_execution"
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/rs/zerolog"
|
||||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type Booking struct {
|
||||||
|
Start time.Time
|
||||||
|
Stop time.Time
|
||||||
|
Duration uint
|
||||||
|
Workflow string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s Booking) Equals(other Booking) bool {
|
||||||
|
return s.Workflow == other.Workflow && s.Start == other.Start && s.Stop == other.Stop
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *Booking) String() string {
|
||||||
|
return fmt.Sprintf("{workflow : %s , start_date : %s , stop_date : %s }", b.Workflow, b.Start.Format(time.RFC3339), b.Stop.Format(time.RFC3339))
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
type ScheduledBooking struct {
|
||||||
|
Bookings []Booking
|
||||||
|
Mu sync.Mutex
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sb *ScheduledBooking) AddSchedule(new_booking Booking, logger zerolog.Logger) {
|
||||||
|
if !sb.scheduleAlreadyExists(new_booking) {
|
||||||
|
sb.Bookings = append(sb.Bookings, new_booking)
|
||||||
|
logger.Info().Msg("Updated list schedules : \n " + sb.String())
|
||||||
|
}
|
||||||
|
logger.Debug().Msg("Workflow received not added")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sb *ScheduledBooking) scheduleAlreadyExists(new_booking Booking) bool {
|
||||||
|
for _, booking := range sb.Bookings {
|
||||||
|
if booking.Equals(new_booking) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sb *ScheduledBooking) String() string {
|
||||||
|
var str string
|
||||||
|
for _, booking := range sb.Bookings {
|
||||||
|
str += fmt.Sprintf("%s\n", booking.String())
|
||||||
|
}
|
||||||
|
return str
|
||||||
|
}
|
||||||
|
|
||||||
// NATS daemon listens to subject " workflowsUpdate "
|
// NATS daemon listens to subject " workflowsUpdate "
|
||||||
// workflowsUpdate messages must be formatted following this pattern '{"workflow" : "", "start_date" : "", "stop_date" : "" }'
|
// workflowsUpdate messages must be formatted following this pattern '{"workflow" : "", "start_date" : "", "stop_date" : "" }'
|
||||||
|
|
||||||
|
|
||||||
type ScheduleManager struct {
|
type ScheduleManager struct {
|
||||||
Api_url string
|
Logger zerolog.Logger
|
||||||
bookings *models.ScheduledBooking
|
bookings ScheduledBooking
|
||||||
ws models.HttpQuery
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *ScheduleManager) SetBookings(b *models.ScheduledBooking){
|
|
||||||
s.bookings = b
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Goroutine listening to a NATS server for updates
|
// Goroutine listening to a NATS server for updates
|
||||||
// on workflows' scheduling. Messages must contain
|
// on workflows' scheduling. Messages must contain
|
||||||
// workflow execution ID, to allow retrieval of execution infos
|
// workflow execution ID, to allow retrieval of execution infos
|
||||||
func (s *ScheduleManager) ListenForWorkflowSubmissions(){
|
func (s *ScheduleManager) ListenForWorkflowSubmissions() {
|
||||||
|
|
||||||
if(s.bookings == nil){
|
|
||||||
logger.Logger.Error().Msg("booking has not been set in the schedule manager")
|
|
||||||
}
|
|
||||||
|
|
||||||
nc, err := nats.Connect(nats.DefaultURL)
|
nc, err := nats.Connect(nats.DefaultURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Logger.Error().Msg("Could not connect to NATS")
|
s.Logger.Error().Msg("Could not connect to NATS")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -49,34 +82,31 @@ func (s *ScheduleManager) ListenForWorkflowSubmissions(){
|
|||||||
|
|
||||||
ch := make(chan *nats.Msg, 64)
|
ch := make(chan *nats.Msg, 64)
|
||||||
|
|
||||||
subs , err := nc.ChanSubscribe("workflowsUpdate", ch)
|
subs, err := nc.ChanSubscribe("workflowsUpdate", ch)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Logger.Error().Msg("Error listening to NATS")
|
s.Logger.Error().Msg("Error listening to NATS")
|
||||||
}
|
}
|
||||||
defer subs.Unsubscribe()
|
defer subs.Unsubscribe()
|
||||||
|
|
||||||
for msg := range(ch){
|
for msg := range ch {
|
||||||
fmt.Println("Waiting...")
|
fmt.Println("Waiting...")
|
||||||
|
|
||||||
map_mess := retrieveMapFromSub(msg.Data)
|
map_mess := retrieveMapFromSub(msg.Data)
|
||||||
|
|
||||||
s.bookings.Mu.Lock()
|
s.bookings.Mu.Lock()
|
||||||
|
|
||||||
wf_exec := getWorkflowExecution(map_mess["workflow"])
|
wf_exec := s.getWorkflowExecution(map_mess["workflow"])
|
||||||
|
|
||||||
s.bookings.AddSchedule(models.Booking{Workflow: map_mess["workflow"], Start: *wf_exec.ExecDate, Stop: *wf_exec.EndDate })
|
s.bookings.AddSchedule(Booking{Workflow: map_mess["workflow"], Start: *wf_exec.ExecDate, Stop: *wf_exec.EndDate}, s.Logger)
|
||||||
s.bookings.Mu.Unlock()
|
s.bookings.Mu.Unlock()
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *ScheduleManager) getWorkflowExecution(exec_id string) *workflow_execution.WorkflowExecution {
|
||||||
func getWorkflowExecution(exec_id string) *workflow_execution.WorkflowExecution {
|
res := oclib.LoadOne(oclib.LibDataEnum(oclib.WORKFLOW_EXECUTION), exec_id)
|
||||||
|
|
||||||
res := oclib.LoadOne(oclib.LibDataEnum(oclib.WORKFLOW_EXECUTION),exec_id)
|
|
||||||
|
|
||||||
if res.Code != 200 {
|
if res.Code != 200 {
|
||||||
logger.Logger.Error().Msg("Could not retrieve workflow ID from execution ID " + exec_id)
|
s.Logger.Error().Msg("Could not retrieve workflow ID from execution ID " + exec_id)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -93,62 +123,57 @@ func retrieveMapFromSub(message []byte) (result_map map[string]string) {
|
|||||||
|
|
||||||
// Used at launch of the component to retrieve the next scheduled workflows
|
// Used at launch of the component to retrieve the next scheduled workflows
|
||||||
// and then every X minutes in case some workflows were scheduled before launch
|
// and then every X minutes in case some workflows were scheduled before launch
|
||||||
func (s *ScheduleManager) SchedulePolling (){
|
func (s *ScheduleManager) SchedulePolling() {
|
||||||
var sleep_time float64 = 1
|
var sleep_time float64 = 1
|
||||||
for(true){
|
for {
|
||||||
s.getNextScheduledWorkflows(3)
|
s.getNextScheduledWorkflows(3)
|
||||||
|
|
||||||
logger.Logger.Info().Msg("Current list of schedules")
|
s.Logger.Info().Msg("Current list of schedules")
|
||||||
fmt.Println(s.bookings.Bookings)
|
fmt.Println(s.bookings.Bookings)
|
||||||
|
|
||||||
time.Sleep(time.Minute * time.Duration(sleep_time))
|
time.Sleep(time.Minute * time.Duration(sleep_time))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
func (s *ScheduleManager) getWorfklowExecution(from time.Time, to time.Time) (exec_list []workflow_execution.WorkflowExecution, err error) {
|
func (s *ScheduleManager) getWorfklowExecution(from time.Time, to time.Time) (exec_list []workflow_execution.WorkflowExecution, err error) {
|
||||||
|
|
||||||
f := dbs.Filters{
|
f := dbs.Filters{
|
||||||
And: map[string][]dbs.Filter{
|
And: map[string][]dbs.Filter{
|
||||||
"execution_date" : {{Operator : dbs.GTE.String(), Value: primitive.NewDateTimeFromTime(from)}, {Operator: dbs.LTE.String(), Value: primitive.NewDateTimeFromTime(to)}},
|
"execution_date": {{Operator: dbs.GTE.String(), Value: primitive.NewDateTimeFromTime(from)}, {Operator: dbs.LTE.String(), Value: primitive.NewDateTimeFromTime(to)}},
|
||||||
"state": {{Operator: dbs.EQUAL.String(), Value: 1}},
|
"state": {{Operator: dbs.EQUAL.String(), Value: 1}},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
res := oclib.Search(&f,"",oclib.LibDataEnum(oclib.WORKFLOW_EXECUTION))
|
res := oclib.Search(&f, "", oclib.LibDataEnum(oclib.WORKFLOW_EXECUTION))
|
||||||
if res.Code != 200 {
|
if res.Code != 200 {
|
||||||
logger.Logger.Error().Msg("Error loading")
|
s.Logger.Error().Msg("Error loading")
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, exec := range(res.Data){
|
for _, exec := range res.Data {
|
||||||
lib_data := oclib.LoadOne(oclib.LibDataEnum(oclib.WORKFLOW_EXECUTION),exec.GetID())
|
lib_data := oclib.LoadOne(oclib.LibDataEnum(oclib.WORKFLOW_EXECUTION), exec.GetID())
|
||||||
exec_obj := lib_data.ToWorkflowExecution()
|
exec_obj := lib_data.ToWorkflowExecution()
|
||||||
exec_list = append(exec_list, *exec_obj)
|
exec_list = append(exec_list, *exec_obj)
|
||||||
}
|
}
|
||||||
return exec_list, nil
|
return exec_list, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO : refactor to implement oclib.Search
|
func (s *ScheduleManager) getNextScheduledWorkflows(minutes float64) {
|
||||||
func (s *ScheduleManager) getNextScheduledWorkflows(minutes float64) {
|
|
||||||
start := time.Now().UTC()
|
start := time.Now().UTC()
|
||||||
end := start.Add(time.Minute * time.Duration(minutes)).UTC()
|
end := start.Add(time.Minute * time.Duration(minutes)).UTC()
|
||||||
|
|
||||||
fmt.Printf("Getting workflows execution from %s to %s \n", start.String(), end.String())
|
fmt.Printf("Getting workflows execution from %s to %s \n", start.String(), end.String())
|
||||||
|
|
||||||
next_wf_exec, err := s.getWorfklowExecution(start,end)
|
next_wf_exec, err := s.getWorfklowExecution(start, end)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Logger.Error().Msg("Could not retrieve next schedules")
|
s.Logger.Error().Msg("Could not retrieve next schedules")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
s.bookings.Mu.Lock()
|
s.bookings.Mu.Lock()
|
||||||
defer s.bookings.Mu.Unlock()
|
defer s.bookings.Mu.Unlock()
|
||||||
|
|
||||||
for _, exec := range(next_wf_exec){
|
for _, exec := range next_wf_exec {
|
||||||
exec_start := exec.ExecDate
|
exec_start := exec.ExecDate
|
||||||
exec_stop := exec.EndDate
|
exec_stop := exec.EndDate
|
||||||
|
|
||||||
s.bookings.AddSchedule(models.Booking{Workflow: exec.UUID, Start: *exec_start, Stop: *exec_stop})
|
s.bookings.AddSchedule(Booking{Workflow: exec.UUID, Start: *exec_start, Stop: *exec_stop}, s.Logger)
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -10,6 +10,7 @@ services:
|
|||||||
- "--debug"
|
- "--debug"
|
||||||
networks:
|
networks:
|
||||||
- scheduler
|
- scheduler
|
||||||
|
- catalog
|
||||||
loki:
|
loki:
|
||||||
image: 'grafana/loki'
|
image: 'grafana/loki'
|
||||||
container_name: loki
|
container_name: loki
|
||||||
@ -34,3 +35,5 @@ services:
|
|||||||
networks:
|
networks:
|
||||||
scheduler:
|
scheduler:
|
||||||
external: true
|
external: true
|
||||||
|
catalog:
|
||||||
|
external: true
|
@ -1,14 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"oc-scheduler/daemons"
|
|
||||||
"oc-scheduler/models"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestCreateManifest(t *testing.T){
|
|
||||||
em := daemons.ExecutionManager{}
|
|
||||||
em.CreateManifest(models.Booking{},"fessity-chlics_23_07_2024_154326")
|
|
||||||
|
|
||||||
|
|
||||||
}
|
|
@ -1,13 +0,0 @@
|
|||||||
package logger
|
|
||||||
|
|
||||||
import (
|
|
||||||
"cloud.o-forge.io/core/oc-lib/logs"
|
|
||||||
"github.com/rs/zerolog"
|
|
||||||
)
|
|
||||||
|
|
||||||
var Logger zerolog.Logger
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
logs.SetAppName("oc-scheduler")
|
|
||||||
Logger = logs.CreateLogger("","")
|
|
||||||
}
|
|
45
main.go
45
main.go
@ -3,54 +3,23 @@ package main
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
conf "oc-scheduler/conf"
|
conf "oc-schedulerd/conf"
|
||||||
"oc-scheduler/models"
|
"oc-schedulerd/daemons"
|
||||||
|
|
||||||
"oc-scheduler/daemons"
|
|
||||||
|
|
||||||
oclib "cloud.o-forge.io/core/oc-lib"
|
oclib "cloud.o-forge.io/core/oc-lib"
|
||||||
)
|
)
|
||||||
|
|
||||||
// var log zerolog.Logger
|
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
var bookings models.ScheduledBooking
|
oclib.SetConfig(conf.GetConfig().MongoUrl, "DC_myDC")
|
||||||
|
oclib.Init("oc-schedulerd")
|
||||||
|
|
||||||
oclib.SetConfig(conf.GetConfig().MongoUrl,"DC_myDC")
|
sch_mngr := daemons.ScheduleManager{Logger: oclib.GetLogger()}
|
||||||
oclib.Init("oc-scheduler")
|
|
||||||
|
|
||||||
app_conf := conf.GetConfig()
|
|
||||||
apiurl := app_conf.OcCatalogUrl
|
|
||||||
|
|
||||||
sch_mngr := daemons.ScheduleManager{Api_url: apiurl}
|
|
||||||
sch_mngr.SetBookings(&bookings)
|
|
||||||
exe_mngr := daemons.ExecutionManager{}
|
exe_mngr := daemons.ExecutionManager{}
|
||||||
exe_mngr.SetBookings(&bookings)
|
|
||||||
|
|
||||||
go sch_mngr.ListenForWorkflowSubmissions()
|
go sch_mngr.ListenForWorkflowSubmissions()
|
||||||
|
|
||||||
go sch_mngr.SchedulePolling()
|
go sch_mngr.SchedulePolling()
|
||||||
|
|
||||||
exe_mngr.RetrieveNextExecutions()
|
exe_mngr.RetrieveNextExecutions()
|
||||||
|
|
||||||
|
fmt.Print("stop")
|
||||||
// method in Schedule manager that checks the first Schedule object for its start date and exe
|
|
||||||
|
|
||||||
// var g Graph
|
|
||||||
|
|
||||||
// list, err := g.GetGraphList(apiurl)
|
|
||||||
// if err != nil {
|
|
||||||
// log.Fatal().Msg("Failed to get the workspaces list, check api url and that api server is up : " + apiurl)
|
|
||||||
// }
|
|
||||||
|
|
||||||
// println("Available workspaces :")
|
|
||||||
// for workspace, _ := range list {
|
|
||||||
// println(workspace)
|
|
||||||
// }
|
|
||||||
|
|
||||||
|
|
||||||
// g.LoadFrom(list["test-alpr"])
|
|
||||||
// g.ExportToArgo("test-alpr")
|
|
||||||
|
|
||||||
fmt.Print("stop")
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -1,53 +0,0 @@
|
|||||||
package models
|
|
||||||
|
|
||||||
import (
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/http/cookiejar"
|
|
||||||
"net/url"
|
|
||||||
)
|
|
||||||
|
|
||||||
type HttpQuery struct {
|
|
||||||
baseurl string
|
|
||||||
jar http.CookieJar
|
|
||||||
Cookies map[string]string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *HttpQuery) Init(url string) {
|
|
||||||
h.baseurl = url
|
|
||||||
h.jar, _ = cookiejar.New(nil)
|
|
||||||
h.Cookies = make(map[string]string)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *HttpQuery) Get(url string) ([]byte, error) {
|
|
||||||
client := &http.Client{Jar: h.jar}
|
|
||||||
resp, err := client.Get(h.baseurl + url)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
// store received cookies
|
|
||||||
for _, cookie := range h.jar.Cookies(resp.Request.URL) {
|
|
||||||
h.Cookies[cookie.Name] = cookie.Value
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
body, err := io.ReadAll(resp.Body)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return body, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *HttpQuery) Post(url string, data url.Values) (*http.Response, error) {
|
|
||||||
client := &http.Client{Jar: h.jar}
|
|
||||||
resp, err := client.PostForm(h.baseurl+url, data)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
// store received cookies
|
|
||||||
for _, cookie := range h.jar.Cookies(resp.Request.URL) {
|
|
||||||
h.Cookies[cookie.Name] = cookie.Value
|
|
||||||
}
|
|
||||||
return resp, err
|
|
||||||
}
|
|
@ -1,71 +0,0 @@
|
|||||||
package models
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"oc-scheduler/logger"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Is duration really important ?
|
|
||||||
|
|
||||||
type Booking struct {
|
|
||||||
Start time.Time
|
|
||||||
Stop time.Time
|
|
||||||
Duration uint
|
|
||||||
Workflow string
|
|
||||||
}
|
|
||||||
|
|
||||||
type ScheduledBooking struct {
|
|
||||||
Bookings []Booking
|
|
||||||
Mu sync.Mutex
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s Booking) Equals(other Booking) bool {
|
|
||||||
return s.Workflow == other.Workflow && s.Start == other.Start && s.Stop == other.Stop
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sb *ScheduledBooking) AddSchedule(new_booking Booking){
|
|
||||||
if(!sb.scheduleAlreadyExists(new_booking)){
|
|
||||||
sb.Bookings = append(sb.Bookings,new_booking)
|
|
||||||
logger.Logger.Info().Msg("Updated list schedules : \n " + sb.String())
|
|
||||||
} else {
|
|
||||||
// Debug condition : delete once this feature is ready to be implemented
|
|
||||||
logger.Logger.Debug().Msg("Workflow received not added")
|
|
||||||
logger.Logger.Debug().Msg("current schedule contains")
|
|
||||||
for _, booking := range(sb.Bookings){
|
|
||||||
logger.Logger.Debug().Msg(booking.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sb *ScheduledBooking) GetListNames()(list_names []string ){
|
|
||||||
for _, schedule := range(sb.Bookings){
|
|
||||||
list_names = append(list_names, schedule.Workflow)
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sb *ScheduledBooking) scheduleAlreadyExists(new_booking Booking) bool {
|
|
||||||
for _, booking := range(sb.Bookings){
|
|
||||||
if booking.Equals(new_booking){
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *Booking) String() string {
|
|
||||||
return fmt.Sprintf("{workflow : %s , start_date : %s , stop_date : %s }", b.Workflow, b.Start.Format(time.RFC3339), b.Stop.Format(time.RFC3339))
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sb *ScheduledBooking) String() string {
|
|
||||||
var str string
|
|
||||||
for _, booking := range(sb.Bookings){
|
|
||||||
str += fmt.Sprintf("%s\n", booking.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
return str
|
|
||||||
}
|
|
@ -1,40 +0,0 @@
|
|||||||
package models
|
|
||||||
|
|
||||||
type Parameter struct {
|
|
||||||
Name string `yaml:"name,omitempty"`
|
|
||||||
Value string `yaml:"value,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type Container struct {
|
|
||||||
Image string `yaml:"image"`
|
|
||||||
Command []string `yaml:"command,omitempty,flow"`
|
|
||||||
Args []string `yaml:"args,omitempty,flow"`
|
|
||||||
VolumeMounts []VolumeMount `yaml:"volumeMounts,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type VolumeMount struct {
|
|
||||||
Name string `yaml:"name"`
|
|
||||||
MountPath string `yaml:"mountPath"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type Task struct {
|
|
||||||
Name string `yaml:"name"`
|
|
||||||
Template string `yaml:"template"`
|
|
||||||
Dependencies []string `yaml:"dependencies,omitempty"`
|
|
||||||
Arguments struct {
|
|
||||||
Parameters []Parameter `yaml:"parameters,omitempty"`
|
|
||||||
} `yaml:"arguments,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type Dag struct {
|
|
||||||
Tasks []Task `yaml:"tasks,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type Template struct {
|
|
||||||
Name string `yaml:"name"`
|
|
||||||
Inputs struct {
|
|
||||||
Parameters []Parameter `yaml:"parameters"`
|
|
||||||
} `yaml:"inputs,omitempty"`
|
|
||||||
Container Container `yaml:"container,omitempty"`
|
|
||||||
Dag Dag `yaml:"dag,omitempty"`
|
|
||||||
}
|
|
@ -1,19 +0,0 @@
|
|||||||
package models
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
type VolumeClaimTemplate struct {
|
|
||||||
Metadata struct {
|
|
||||||
Name string `yaml:"name"`
|
|
||||||
} `yaml:"metadata"`
|
|
||||||
Spec VolumeSpec `yaml:"spec"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type VolumeSpec struct {
|
|
||||||
AccessModes []string `yaml:"accessModes,flow"`
|
|
||||||
Resources struct {
|
|
||||||
Requests struct {
|
|
||||||
Storage string `yaml:"storage"`
|
|
||||||
} `yaml:"requests"`
|
|
||||||
} `yaml:"resources"`
|
|
||||||
}
|
|
BIN
oc-scheduler
Executable file
BIN
oc-scheduler
Executable file
Binary file not shown.
Loading…
Reference in New Issue
Block a user