core/oc-scheduler
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

oc-scheduler -> scheduling + logs

Browse Source
This commit is contained in:
mr
2026-04-08 10:05:27 +02:00
parent f8a6e69ef3
commit 1d63d31442
21 changed files with 4605 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
controllers/booking.go
View File

@@ -2,6 +2,7 @@ package controllers
import (
"net/http"
"strconv"
"time"
oclib "cloud.o-forge.io/core/oc-lib"
@@ -23,6 +24,8 @@ var BookingExample booking.Booking
// @Description search bookings by execution
// @Param id path string true "id execution"
// @Param is_draft query string false "draft wished"
// @Param offset query string false
// @Param limit query string false
// @Success 200 {workspace} models.workspace
// @router /search/execution/:id [get]
func (o *BookingController) ExecutionSearch() {
@@ -39,12 +42,14 @@ func (o *BookingController) ExecutionSearch() {
user, peerID, groups := oclib.ExtractTokenInfo(*o.Ctx.Request)
id := o.Ctx.Input.Param(":id")
isDraft := o.Ctx.Input.Query("is_draft")
offset, _ := strconv.Atoi(o.Ctx.Input.Query("offset"))
limit, _ := strconv.Atoi(o.Ctx.Input.Query("limit"))
f := dbs.Filters{
Or: map[string][]dbs.Filter{ // filter by name if no filters are provided
"execution_id": {{Operator: dbs.EQUAL.String(), Value: id}},
},
}
o.Data["json"] = oclib.NewRequest(oclib.LibDataEnum(oclib.BOOKING), user, peerID, groups, nil).Search(&f, "", isDraft == "true")
o.Data["json"] = oclib.NewRequest(oclib.LibDataEnum(oclib.BOOKING), user, peerID, groups, nil).Search(&f, "", isDraft == "true", int64(offset), int64(limit))
o.ServeJSON()
}
@@ -53,6 +58,8 @@ func (o *BookingController) ExecutionSearch() {
// @Param start_date path string true "the word search you want to get"
// @Param end_date path string true "the word search you want to get"
// @Param is_draft query string false "draft wished"
// @Param offset query string false
// @Param limit query string false
// @Success 200 {workspace} models.workspace
// @router /search/:start_date/:end_date [get]
func (o *BookingController) Search() {
@@ -67,6 +74,8 @@ func (o *BookingController) Search() {
*/
// store and return Id or post with UUID
user, peerID, groups := oclib.ExtractTokenInfo(*o.Ctx.Request)
offset, _ := strconv.Atoi(o.Ctx.Input.Query("offset"))
limit, _ := strconv.Atoi(o.Ctx.Input.Query("limit"))
start_date, _ := time.ParseInLocation("2006-01-02", o.Ctx.Input.Param(":start_date"), time.UTC)
end_date, _ := time.ParseInLocation("2006-01-02", o.Ctx.Input.Param(":end_date"), time.UTC)
isDraft := o.Ctx.Input.Query("is_draft")
@@ -77,19 +86,23 @@ func (o *BookingController) Search() {
"execution_date": {{Operator: "gte", Value: sd}, {Operator: "lte", Value: ed}},
},
}
o.Data["json"] = oclib.NewRequest(oclib.LibDataEnum(oclib.BOOKING), user, peerID, groups, nil).Search(&f, "", isDraft == "true")
o.Data["json"] = oclib.NewRequest(oclib.LibDataEnum(oclib.BOOKING), user, peerID, groups, nil).Search(&f, "", isDraft == "true", int64(offset), int64(limit))
o.ServeJSON()
}
// @Title GetAll
// @Description find booking by id
// @Param is_draft query string false "draft wished"
// @Param offset query string false
// @Param limit query string false
// @Success 200 {booking} models.booking
// @router / [get]
func (o *BookingController) GetAll() {
user, peerID, groups := oclib.ExtractTokenInfo(*o.Ctx.Request)
offset, _ := strconv.Atoi(o.Ctx.Input.Query("offset"))
limit, _ := strconv.Atoi(o.Ctx.Input.Query("limit"))
isDraft := o.Ctx.Input.Query("is_draft")
o.Data["json"] = oclib.NewRequest(oclib.LibDataEnum(oclib.BOOKING), user, peerID, groups, nil).LoadAll(isDraft == "true")
o.Data["json"] = oclib.NewRequest(oclib.LibDataEnum(oclib.BOOKING), user, peerID, groups, nil).LoadAll(isDraft == "true", int64(offset), int64(limit))
o.ServeJSON()
}

7
controllers/execution_verification.go
View File

@@ -3,6 +3,7 @@ package controllers
import (
"encoding/json"
"oc-scheduler/infrastructure"
"strconv"
oclib "cloud.o-forge.io/core/oc-lib"
"cloud.o-forge.io/core/oc-lib/models/execution_verification"
@@ -19,12 +20,16 @@ type ExecutionVerificationController struct {
// @Title GetAll
// @Description find verification by id
// @Param is_draft query string false "draft wished"
// @Param offset query string false
// @Param limit query string false
// @Success 200 {booking} models.booking
// @router / [get]
func (o *ExecutionVerificationController) GetAll() {
user, peerID, groups := oclib.ExtractTokenInfo(*o.Ctx.Request)
isDraft := o.Ctx.Input.Query("is_draft")
o.Data["json"] = oclib.NewRequest(oclib.LibDataEnum(oclib.EXECUTION_VERIFICATION), user, peerID, groups, nil).LoadAll(isDraft == "true")
offset, _ := strconv.Atoi(o.Ctx.Input.Query("offset"))
limit, _ := strconv.Atoi(o.Ctx.Input.Query("limit"))
o.Data["json"] = oclib.NewRequest(oclib.LibDataEnum(oclib.EXECUTION_VERIFICATION), user, peerID, groups, nil).LoadAll(isDraft == "true", int64(offset), int64(limit))
o.ServeJSON()
}

90
controllers/loki.go
View File

@@ -5,10 +5,12 @@ import (
"fmt"
"io"
"net/http"
"net/url"
"strings"
"cloud.o-forge.io/core/oc-lib/config"
beego "github.com/beego/beego/v2/server/web"
gorillaws "github.com/gorilla/websocket"
)
// Operations about workflow
@@ -84,3 +86,91 @@ func (o *LokiController) GetLogs() {
o.Data["json"] = map[string]string{"error": "Query error"}
o.ServeJSON()
}
// LogsStreamHandler streams Loki logs over WebSocket.
//
// The client sends one JSON message with the same format as GetLogs:
//
// {"start": "<unix-seconds>", "label1": "val1", ...}
//
// The server connects to Loki's /loki/api/v1/tail WebSocket endpoint and
// forwards every message it receives until the client disconnects.
func LogsStreamHandler(w http.ResponseWriter, r *http.Request) {
conn, err := wsUpgrader.Upgrade(w, r, nil)
if err != nil {
return
}
defer conn.Close()
var query map[string]interface{}
if err := conn.ReadJSON(&query); err != nil {
return
}
start := fmt.Sprintf("%v", query["start"])
if len(start) > 10 {
start = start[:10]
}
var labels []string
for k, v := range query {
if k == "start" || k == "end" {
continue
}
labels = append(labels, fmt.Sprintf("%v=\"%v\"", k, v))
}
if len(labels) == 0 || len(start) < 10 {
_ = conn.WriteJSON(map[string]string{"error": "missing start or query labels"})
return
}
// Build Loki tail WS URL (http→ws, https→wss).
lokiBase := config.GetConfig().LokiUrl
lokiBase = strings.Replace(lokiBase, "https://", "wss://", 1)
lokiBase = strings.Replace(lokiBase, "http://", "ws://", 1)
lokiURL := lokiBase + "/loki/api/v1/tail?" + url.Values{
"query": {"{" + strings.Join(labels, ", ") + "}"},
"start": {start + "000000000"}, // seconds → nanoseconds
}.Encode()
lokiConn, _, err := gorillaws.DefaultDialer.Dial(lokiURL, nil)
if err != nil {
_ = conn.WriteJSON(map[string]string{"error": "loki: " + err.Error()})
return
}
defer lokiConn.Close()
errCh := make(chan error, 2)
// Forward Loki → client.
go func() {
for {
_, msg, err := lokiConn.ReadMessage()
if err != nil {
errCh <- err
return
}
var result map[string]interface{}
if json.Unmarshal(msg, &result) == nil {
if err := conn.WriteJSON(result); err != nil {
errCh <- err
return
}
}
}
}()
// Detect client disconnect (read pump).
go func() {
for {
if _, _, err := conn.ReadMessage(); err != nil {
errCh <- err
return
}
}
}()
<-errCh
}

178
controllers/sheduler.go
View File

@@ -1,11 +1,15 @@
package controllers
import (
"context"
"fmt"
"net/http"
"oc-scheduler/infrastructure"
"reflect"
"strconv"
"strings"
"sync"
"time"
oclib "cloud.o-forge.io/core/oc-lib"
"cloud.o-forge.io/core/oc-lib/dbs"
@@ -27,12 +31,66 @@ var wsUpgrader = gorillaws.Upgrader{
CheckOrigin: func(r *http.Request) bool { return true },
}
var schedulerMu sync.RWMutex
var scheduler = map[string]*infrastructure.WorkflowSchedule{}
func realPushCheckfunc(ctx context.Context, conn *gorillaws.Conn, req *tools.APIRequest, user string, ws infrastructure.WorkflowSchedule,
executionsID string, wfID string, scheduled bool, asap bool, preemption bool, reschedule bool) (bool, error) {
// If we already have draft bookings for this session and we're about to
// re-check (timer refresh or planner update), remove the old drafts first
// so the planner doesn't treat our own previous reservations as conflicts.
if reschedule && scheduled {
infrastructure.CleanupSession(executionsID, req)
scheduled = false
}
workflowScheduler := ws
schedulerMu.Lock()
if scheduler[user] != nil {
workflowScheduler = *scheduler[user]
}
schedulerMu.Unlock()
result, checkErr := workflowScheduler.Check(wfID, asap, preemption, req)
fmt.Println("CHECK", checkErr)
if checkErr != nil {
return scheduled, checkErr
}
if result.Available && reschedule {
workflowScheduler.Start = result.Start
if result.End != nil {
workflowScheduler.End = result.End
}
_, _, execs, purchases, bookings, err := workflowScheduler.GetBuyAndBook(wfID, req)
if err != nil {
fmt.Println("GetBuyAndBook", err)
return scheduled, err
}
infrastructure.UpsertSessionDrafts(executionsID, execs, purchases, bookings, req)
scheduled = true
delay := workflowScheduler.Start.UTC().Add(-(1 * time.Minute)).Sub(time.Now().UTC())
go func() {
select {
case <-ctx.Done():
// Session closed before timer fired — nothing to do, CleanupSession
// has already run (or will run) in the defer of CheckStreamHandler.
return
case <-time.After(delay):
realPushCheckfunc(ctx, conn, req, user, ws, executionsID, wfID, scheduled, asap, preemption, true)
}
}()
}
result.SchedulingID = executionsID
fmt.Println(result)
return scheduled, conn.WriteJSON(result)
}
// CheckStreamHandler is the WebSocket handler for slot availability checking.
// Query params: as_possible=true, preemption=true
func CheckStreamHandler(w http.ResponseWriter, r *http.Request) {
var err error
wfID := strings.TrimSuffix(
strings.TrimPrefix(r.URL.Path, "/oc/"),
"/check",
strings.TrimPrefix(r.URL.Path, "/oc/check/"),
"",
)
q := r.URL.Query()
@@ -49,22 +107,31 @@ func CheckStreamHandler(w http.ResponseWriter, r *http.Request) {
}
watchedPeers, err := infrastructure.GetWorkflowPeerIDs(wfID, req)
fmt.Println("Watched peers for workflow", wfID, ":", watchedPeers)
fmt.Println("Watched peers for workflow", wfID, ":", watchedPeers, err)
if err != nil {
http.Error(w, `{"code":404,"error":"`+err.Error()+`"}`, http.StatusNotFound)
return
}
conn, err := wsUpgrader.Upgrade(w, r, nil)
fmt.Println("Upgrade :", err)
if err != nil {
return
}
var ws infrastructure.WorkflowSchedule
if err := conn.ReadJSON(&ws); err != nil {
fmt.Println("ReadJSON :", err)
conn.Close()
return
}
// Allow the initial JSON to override the query-param mode.
if ws.Asap != nil {
asap = *ws.Asap
}
if ws.Preemption != nil {
preemption = *ws.Preemption
}
plannerCh, plannerUnsub := infrastructure.SubscribePlannerUpdates(watchedPeers)
wfCh, wfUnsub := infrastructure.SubscribeWorkflowUpdates(wfID)
@@ -83,7 +150,9 @@ func CheckStreamHandler(w http.ResponseWriter, r *http.Request) {
scheduled := false
confirmed := false
ctx, cancel := context.WithCancel(context.Background())
defer func() {
cancel()
conn.Close()
plannerUnsub()
wfUnsub()
@@ -93,28 +162,9 @@ func CheckStreamHandler(w http.ResponseWriter, r *http.Request) {
}
}()
pushCheck := func(reschedule bool) error {
result, checkErr := ws.Check(wfID, asap, preemption, req)
if checkErr != nil {
return checkErr
}
if result.Available && reschedule {
ws.Start = result.Start
if result.End != nil {
ws.End = result.End
}
_, _, execs, purchases, bookings, err := ws.GetBuyAndBook(wfID, req)
if err != nil {
return err
}
infrastructure.UpsertSessionDrafts(executionsID, execs, purchases, bookings, req)
scheduled = true
}
result.SchedulingID = executionsID
return conn.WriteJSON(result)
}
if err := pushCheck(true); err != nil {
pushCheck := realPushCheckfunc
if scheduled, err = pushCheck(ctx, conn, req, user, ws, executionsID, wfID, scheduled, asap, preemption, true); err != nil {
fmt.Println("UPDATE CONFIRM FIRST scheduled", err)
return
}
@@ -139,9 +189,17 @@ func CheckStreamHandler(w http.ResponseWriter, r *http.Request) {
for {
select {
case updated := <-updateCh:
fmt.Println("updated FOUND ", updated)
workflowScheduler := ws
schedulerMu.Lock()
if scheduler[user] != nil {
workflowScheduler = *scheduler[user]
}
schedulerMu.Unlock()
if updated.Confirm {
ws.UUID = executionsID
_, _, _, schedErr := infrastructure.Schedule(&ws, wfID, req)
workflowScheduler.UUID = executionsID
_, _, _, schedErr := infrastructure.Schedule(&workflowScheduler, wfID, req)
if schedErr != nil {
_ = conn.WriteJSON(map[string]interface{}{
"error": schedErr.Error(),
@@ -149,38 +207,62 @@ func CheckStreamHandler(w http.ResponseWriter, r *http.Request) {
return
}
confirmed = true
fmt.Println("UPDATE CONFIRM")
return
}
changed := updated.Cron != ws.Cron ||
!updated.Start.Equal(ws.Start) ||
updated.DurationS != ws.DurationS ||
(updated.End == nil) != (ws.End == nil) ||
(updated.End != nil && ws.End != nil && !updated.End.Equal(*ws.End)) ||
updated.BookingMode != ws.BookingMode ||
!reflect.DeepEqual(updated.SelectedBillingStrategy, ws.SelectedBillingStrategy) ||
!reflect.DeepEqual(updated.SelectedInstances, ws.SelectedInstances) ||
!reflect.DeepEqual(updated.SelectedPartnerships, ws.SelectedPartnerships) ||
!reflect.DeepEqual(updated.SelectedBuyings, ws.SelectedBuyings) ||
!reflect.DeepEqual(updated.SelectedStrategies, ws.SelectedStrategies)
// Detect mode change before updating local vars.
modeChanged := (updated.Asap != nil && *updated.Asap != asap) ||
(updated.Preemption != nil && *updated.Preemption != preemption)
if updated.Asap != nil {
asap = *updated.Asap
}
if updated.Preemption != nil {
preemption = *updated.Preemption
}
changed := modeChanged ||
updated.Cron != workflowScheduler.Cron ||
!updated.Start.Equal(workflowScheduler.Start) ||
updated.DurationS != workflowScheduler.DurationS ||
(updated.End == nil) != (workflowScheduler.End == nil) ||
(updated.End != nil && workflowScheduler.End != nil && !updated.End.Equal(*workflowScheduler.End)) ||
updated.BookingMode != workflowScheduler.BookingMode ||
!reflect.DeepEqual(updated.SelectedBillingStrategy, workflowScheduler.SelectedBillingStrategy) ||
!reflect.DeepEqual(updated.SelectedInstances, workflowScheduler.SelectedInstances) ||
!reflect.DeepEqual(updated.SelectedPartnerships, workflowScheduler.SelectedPartnerships) ||
!reflect.DeepEqual(updated.SelectedBuyings, workflowScheduler.SelectedBuyings) ||
!reflect.DeepEqual(updated.SelectedStrategies, workflowScheduler.SelectedStrategies)
infrastructure.CleanupSession(executionsID, req)
ws = updated
if err := pushCheck(changed || !scheduled); err != nil {
schedulerMu.Lock()
scheduler[user] = &updated
schedulerMu.Unlock()
if scheduled, err = pushCheck(ctx, conn, req, user, ws, executionsID, wfID, scheduled, asap, preemption, changed || !scheduled); err != nil {
fmt.Println("UPDATE SCHEDULERD", err)
return
}
case remotePeerID := <-plannerCh:
workflowScheduler := ws
schedulerMu.Lock()
if scheduler[user] != nil {
workflowScheduler = *scheduler[user]
}
schedulerMu.Unlock()
if remotePeerID == selfPeerID {
if scheduled {
continue
}
result, checkErr := ws.Check(wfID, asap, preemption, req)
result, checkErr := workflowScheduler.Check(wfID, asap, preemption, req)
if checkErr == nil {
result.SchedulingID = executionsID
_ = conn.WriteJSON(result)
}
continue
}
if err := pushCheck(scheduled); err != nil {
if scheduled, err = pushCheck(ctx, conn, req, user, ws, executionsID, wfID, scheduled, asap, preemption, scheduled); err != nil {
fmt.Println("UPDATE SCHEDULERD PLAN", err)
return
}
@@ -192,11 +274,13 @@ func CheckStreamHandler(w http.ResponseWriter, r *http.Request) {
newOwned := infrastructure.RequestPlannerRefresh(newPeers, executionsID)
ownedPeers = append(ownedPeers, newOwned...)
}
if err := pushCheck(false); err != nil {
if scheduled, err = pushCheck(ctx, conn, req, user, ws, executionsID, wfID, scheduled, asap, preemption, false); err != nil {
fmt.Println("UPDATE WORKFLOW", err)
return
}
case <-closeCh:
fmt.Println("UPDATE Close ? ")
return
}
}
@@ -227,9 +311,13 @@ func (o *WorkflowSchedulerController) UnSchedule() {
// @Title SearchScheduledDraftOrder
// @Description search draft order for a workflow
// @Param id path string true "id execution"
// @Param offset query string false
// @Param limit query string false
// @Success 200 {workspace} models.workspace
// @router /:id/order [get]
// @router /order/:id [get]
func (o *WorkflowSchedulerController) SearchScheduledDraftOrder() {
offset, _ := strconv.Atoi(o.Ctx.Input.Query("offset"))
limit, _ := strconv.Atoi(o.Ctx.Input.Query("limit"))
_, peerID, _ := oclib.ExtractTokenInfo(*o.Ctx.Request)
id := o.Ctx.Input.Param(":id")
filter := &dbs.Filters{
@@ -238,6 +326,6 @@ func (o *WorkflowSchedulerController) SearchScheduledDraftOrder() {
"order_by": {{Operator: dbs.EQUAL.String(), Value: peerID}},
},
}
o.Data["json"] = oclib.NewRequestAdmin(orderCollection, nil).Search(filter, "", true)
o.Data["json"] = oclib.NewRequestAdmin(orderCollection, nil).Search(filter, "", true, int64(offset), int64(limit))
o.ServeJSON()
}

26
controllers/workflow_execution.go
View File

@@ -1,6 +1,7 @@
package controllers
import (
"strconv"
"time"
oclib "cloud.o-forge.io/core/oc-lib"
@@ -21,6 +22,8 @@ type WorkflowExecutionController struct {
// @Param start_date path string true "the word search you want to get"
// @Param end_date path string true "the word search you want to get"
// @Param is_draft query string false "draft wished"
// @Param offset query string false
// @Param limit query string false
// @Success 200 {workspace} models.workspace
// @router /search/:start_date/:end_date [get]
func (o *WorkflowExecutionController) SearchPerDate() {
@@ -35,6 +38,8 @@ func (o *WorkflowExecutionController) SearchPerDate() {
*/
// user, peerID, groups := oclib.ExtractTokenInfo(*o.Ctx.Request)
// store and return Id or post with UUID
offset, _ := strconv.Atoi(o.Ctx.Input.Query("offset"))
limit, _ := strconv.Atoi(o.Ctx.Input.Query("limit"))
start_date, _ := time.ParseInLocation("2006-01-02", o.Ctx.Input.Param(":start_date"), time.UTC)
end_date, _ := time.ParseInLocation("2006-01-02", o.Ctx.Input.Param(":end_date"), time.UTC)
sd := primitive.NewDateTimeFromTime(start_date)
@@ -46,7 +51,7 @@ func (o *WorkflowExecutionController) SearchPerDate() {
}
isDraft := o.Ctx.Input.Query("is_draft")
// o.Data["json"] = oclib.NewRequest(collection, user, peerID, groups, nil).Search(&f, "", isDraft == "true")
o.Data["json"] = oclib.NewRequestAdmin(collection, nil).Search(&f, "", isDraft == "true")
o.Data["json"] = oclib.NewRequestAdmin(collection, nil).Search(&f, "", isDraft == "true", int64(offset), int64(limit))
o.ServeJSON()
}
@@ -54,13 +59,16 @@ func (o *WorkflowExecutionController) SearchPerDate() {
// @Title GetAll
// @Description find workflow by workflowid
// @Param is_draft query string false "draft wished"
// @Param offset query string false
// @Param limit query string false
// @Success 200 {workflow} models.workflow
// @router / [get]
func (o *WorkflowExecutionController) GetAll() {
// user, peerID, groups := oclib.ExtractTokenInfo(*o.Ctx.Request)
offset, _ := strconv.Atoi(o.Ctx.Input.Query("offset"))
limit, _ := strconv.Atoi(o.Ctx.Input.Query("limit"))
user, peerID, groups := oclib.ExtractTokenInfo(*o.Ctx.Request)
isDraft := o.Ctx.Input.Query("is_draft")
// o.Data["json"] = oclib.NewRequest(collection, user, peerID, groups, nil).LoadAll(isDraft == "true")
o.Data["json"] = oclib.NewRequestAdmin(collection, nil).LoadAll(isDraft == "true")
o.Data["json"] = oclib.NewRequest(collection, user, peerID, groups, nil).LoadAll(isDraft == "true", int64(offset), int64(limit))
o.ServeJSON()
}
@@ -81,14 +89,16 @@ func (o *WorkflowExecutionController) Get() {
// @Description find compute by key word
// @Param search path string true "the search you want to get"
// @Param is_draft query string false "draft wished"
// @Param offset query string false
// @Param limit query string false
// @Success 200 {compute} models.compute
// @router /search/:search [get]
func (o *WorkflowExecutionController) Search() {
// user, peerID, groups := oclib.ExtractTokenInfo(*o.Ctx.Request)
user, peerID, groups := oclib.ExtractTokenInfo(*o.Ctx.Request)
offset, _ := strconv.Atoi(o.Ctx.Input.Query("offset"))
limit, _ := strconv.Atoi(o.Ctx.Input.Query("limit"))
isDraft := o.Ctx.Input.Query("is_draft")
search := o.Ctx.Input.Param(":search")
// o.Data["json"] = oclib.NewRequest(collection, user, peerID, groups, nil).Search(nil, search, isDraft == "true")
o.Data["json"] = oclib.NewRequestAdmin(collection, nil).Search(nil, search, isDraft == "true")
o.Data["json"] = oclib.NewRequest(collection, user, peerID, groups, nil).Search(nil, search, isDraft == "true", int64(offset), int64(limit))
o.ServeJSON()
}