core/oc-lib
6
0
Fork 0
Code Issues 7 Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: core:main
Branches Tags
core:main
core:bugfix/booking-too-long
core:bugfix/lag-booking
core:bugfix/schedule-cron
core:test-wf
core:feature/admiralty
core:debug/check-peer
core:namespace
core:payment
core:issue#4
core:chart
core:logs
..
compare: core:feature/admiralty
Branches Tags
core:main
core:bugfix/booking-too-long
core:bugfix/lag-booking
core:bugfix/schedule-cron
core:test-wf
core:feature/admiralty
core:debug/check-peer
core:namespace
core:payment
core:issue#4
core:chart
core:logs

No commits in common. "main" and "feature/admiralty" have entirely different histories.
main ... feature/ad

6 changed files with 17 additions and 100 deletions
Show Stats Expand all files Collapse all files

2
dbs/mongo/mongo.go
View File

@ -287,7 +287,7 @@ func (m *MongoDB) Search(filters *dbs.Filters, collection_name string) (*mongo.C
return nil, 503, err
}
opts := options.Find()
opts.SetLimit(1000)
opts.SetLimit(100)
targetDBCollection := CollectionMap[collection_name]
orList := bson.A{}
andList := bson.A{}

4
models/peer/peer_cache.go
View File

@ -93,10 +93,10 @@ func (p *PeerCache) LaunchPeerExecution(peerID string, dataID string,
}
mypeer.AddExecution(*pexec)
NewShallowAccessor().UpdateOne(mypeer, peerID) // Update the peer in the db
return nil, errors.New("peer is " + peerID + " not reachable")
return nil, errors.New("peer is not reachable")
} else {
if mypeer == nil {
return nil, errors.New("peer " + peerID + " not found")
return nil, errors.New("peer not found")
}
// If the peer is reachable, launch the execution
url = p.urlFormat((mypeer.Url), dt) + path // Format the URL

2
models/workflow_execution/workflow_execution.go
View File

@ -112,8 +112,6 @@ func (d *WorkflowExecution) VerifyAuth(request *tools.APIRequest) bool {
func (d *WorkflowExecution) Book(executionsID string, wfID string, priceds map[tools.DataType]map[string]pricing.PricedItemITF) []*booking.Booking {
booking := d.bookEach(executionsID, wfID, tools.STORAGE_RESOURCE, priceds[tools.STORAGE_RESOURCE])
booking = append(booking, d.bookEach(executionsID, wfID, tools.PROCESSING_RESOURCE, priceds[tools.PROCESSING_RESOURCE])...)
booking = append(booking,d.bookEach(executionsID, wfID, tools.COMPUTE_RESOURCE, priceds[tools.COMPUTE_RESOURCE])...)
booking = append(booking,d.bookEach(executionsID, wfID, tools.DATA_RESOURCE, priceds[tools.DATA_RESOURCE])...)
return booking
}

95
models/workflow_execution/workflow_scheduler.go
View File

@ -4,7 +4,6 @@ import (
"errors"
"fmt"
"strings"
"sync"
"time"
"cloud.o-forge.io/core/oc-lib/models/booking"
@ -78,60 +77,22 @@ func (ws *WorkflowSchedule) CheckBooking(wfID string, request *tools.APIRequest)
bookings := []*booking.Booking{}
for _, exec := range execs {
bookings = append(bookings, exec.Book(ws.UUID, wfID, priceds)...)
}
errCh := make(chan error, len(bookings))
var m sync.Mutex
for _, b := range bookings {
go getBooking(b, request, wf, execs, bookings, errCh, &m)
}
for i := 0; i < len(bookings); i++ {
if err := <-errCh; err != nil {
return false, wf, execs, bookings, err
for _, b := range bookings {
meth := request.Caller.URLS[tools.BOOKING][tools.GET]
meth = strings.ReplaceAll(meth, ":id", b.ResourceID)
meth = strings.ReplaceAll(meth, ":start_date", b.ExpectedStartDate.Format("2006-01-02T15:04:05"))
meth = strings.ReplaceAll(meth, ":end_date", b.ExpectedEndDate.Format("2006-01-02T15:04:05"))
request.Caller.URLS[tools.BOOKING][tools.GET] = meth
_, err := (&peer.Peer{}).LaunchPeerExecution(b.DestPeerID, b.ResourceID, tools.BOOKING, tools.GET, nil, request.Caller)
if err != nil {
return false, wf, execs, bookings, err
}
}
}
}
return true, wf, execs, bookings, nil
}
func getBooking( b *booking.Booking, request *tools.APIRequest, wf *workflow.Workflow, execs []*WorkflowExecution, bookings []*booking.Booking, errCh chan error, m *sync.Mutex) {
m.Lock()
c, err := getCallerCopy(request, errCh)
if err != nil {
errCh <- err
return
}
m.Unlock()
meth := c.URLS[tools.BOOKING][tools.GET]
meth = strings.ReplaceAll(meth, ":id", b.ResourceID)
meth = strings.ReplaceAll(meth, ":start_date", b.ExpectedStartDate.Format("2006-01-02T15:04:05"))
meth = strings.ReplaceAll(meth, ":end_date", b.ExpectedEndDate.Format("2006-01-02T15:04:05"))
c.URLS[tools.BOOKING][tools.GET] = meth
_, err = (&peer.Peer{}).LaunchPeerExecution(b.DestPeerID, b.ResourceID, tools.BOOKING, tools.GET, nil, &c)
if err != nil {
errCh <- err
return
}
errCh <- nil
}
func getCallerCopy(request *tools.APIRequest, errCh chan error) (tools.HTTPCaller, error) {
var c tools.HTTPCaller
err := request.Caller.DeepCopy(c)
if err != nil {
errCh <- err
return tools.HTTPCaller{}, nil
}
c.URLS = request.Caller.URLS
return c, err
}
func (ws *WorkflowSchedule) Schedules(wfID string, request *tools.APIRequest) (*WorkflowSchedule, *workflow.Workflow, []*WorkflowExecution, error) {
if request == nil {
return ws, nil, []*WorkflowExecution{}, errors.New("no request found")
@ -150,20 +111,13 @@ func (ws *WorkflowSchedule) Schedules(wfID string, request *tools.APIRequest) (*
return ws, nil, executions, errors.New("could not book the workflow : " + fmt.Sprintf("%v", err))
}
ws.Workflow = wf
var errCh = make(chan error, len(bookings))
var m sync.Mutex
for _, booking := range bookings {
go ws.BookExecs(booking, request, errCh, &m)
}
for i := 0; i < len(bookings); i++ {
if err := <- errCh ; err != nil {
_, err := (&peer.Peer{}).LaunchPeerExecution(booking.DestPeerID, "",
tools.BOOKING, tools.POST, booking.Serialize(booking), request.Caller)
if err != nil {
return ws, wf, executions, errors.New("could not launch the peer execution : " + fmt.Sprintf("%v", err))
}
}
fmt.Println("Schedules")
for _, exec := range executions {
err := exec.PurgeDraft(request)
@ -177,27 +131,6 @@ func (ws *WorkflowSchedule) Schedules(wfID string, request *tools.APIRequest) (*
return ws, wf, executions, nil
}
func (ws *WorkflowSchedule) BookExecs(booking *booking.Booking, request *tools.APIRequest, errCh chan error, m *sync.Mutex) {
m.Lock()
c, err := getCallerCopy(request, errCh)
if err != nil {
errCh <- err
return
}
m.Unlock()
_, err = (&peer.Peer{}).LaunchPeerExecution(booking.DestPeerID, "",
tools.BOOKING, tools.POST, booking.Serialize(booking), &c)
if err != nil {
errCh <- err
return
}
errCh <- nil
}
/*
BOOKING IMPLIED TIME, not of subscription but of execution
so is processing time execution time applied on computes

4
tools/api.go
View File

@ -7,7 +7,6 @@ import (
"cloud.o-forge.io/core/oc-lib/config"
"cloud.o-forge.io/core/oc-lib/dbs/mongo"
"cloud.o-forge.io/core/oc-lib/logs"
beego "github.com/beego/beego/v2/server/web"
)
@ -136,7 +135,6 @@ func (a *API) CheckRemotePeer(url string) (State, map[string]int) {
// CheckRemoteAPIs checks the state of remote APIs from your proper OC
func (a *API) CheckRemoteAPIs(apis []DataType) (State, map[string]string, error) {
// Check if the database is up
l := logs.GetLogger()
new := map[string]string{}
caller := NewHTTPCaller(map[DataType]map[METHOD]string{}) // Create a new http caller
code := 0
@ -147,7 +145,6 @@ func (a *API) CheckRemoteAPIs(apis []DataType) (State, map[string]string, error)
var resp APIStatusResponse
b, err := caller.CallGet("http://"+api.API()+":8080", "/oc/version/status") // Call the status endpoint of the remote API (standard OC status endpoint)
if err != nil {
l.Error().Msg(api.String() + " not reachable")
state = REDUCED_SERVICE // If a remote API is not reachable, return reduced service
continue
}
@ -164,7 +161,6 @@ func (a *API) CheckRemoteAPIs(apis []DataType) (State, map[string]string, error)
reachable = true // If the remote API is reachable, set reachable to true cause we are not dead
}
if !reachable {
l.Error().Msg("Peer check returned no answers")
state = DEAD // If no remote API is reachable, return dead, nobody is alive
}
if code > 0 {

10
tools/remote_caller.go
View File

@ -63,16 +63,6 @@ func NewHTTPCaller(urls map[DataType]map[METHOD]string) *HTTPCaller {
}
}
// Creates a copy of the current caller, in order to have parallelized executions without race condition
func (c* HTTPCaller) DeepCopy(dst HTTPCaller) error {
bytes, err := json.Marshal(c)
if err != nil {
return err
}
return json.Unmarshal(bytes, &dst)
}
// CallGet calls the GET method on the HTTP server
func (caller *HTTPCaller) CallGet(url string, subpath string, types ...string) ([]byte, error) {
req, err := http.NewRequest(http.MethodGet, url+subpath, bytes.NewBuffer([]byte("")))