diff --git a/dbs/dbs.go b/dbs/dbs.go index fc6885d..ff1bd42 100644 --- a/dbs/dbs.go +++ b/dbs/dbs.go @@ -102,12 +102,12 @@ func GetBson(filters *Filters) bson.D { } } if len(orList) > 0 && len(andList) == 0 { - f = bson.D{{"$or", orList}} + f = bson.D{{Key: "$or", Value: orList}} } else { if len(orList) > 0 { andList = append(andList, bson.M{"$or": orList}) } - f = bson.D{{"$and", andList}} + f = bson.D{{Key: "$and", Value: andList}} } } return f diff --git a/dbs/mongo/mongo.go b/dbs/mongo/mongo.go index 9390cf4..43fd855 100644 --- a/dbs/mongo/mongo.go +++ b/dbs/mongo/mongo.go @@ -282,12 +282,11 @@ func (m *MongoDB) LoadOne(id string, collection_name string) (*mongo.SingleResul return res, 200, nil } -func (m *MongoDB) Search(filters *dbs.Filters, collection_name string) (*mongo.Cursor, int, error) { +func (m *MongoDB) Search(filters *dbs.Filters, collection_name string, offset int64, limit int64) (*mongo.Cursor, int, error) { if err := m.createClient(mngoConfig.GetUrl(), false); err != nil { return nil, 503, err } opts := options.Find() - opts.SetLimit(1000) targetDBCollection := CollectionMap[collection_name] if targetDBCollection == nil { return nil, 503, errors.New("collection " + collection_name + " not initialized") @@ -295,6 +294,9 @@ func (m *MongoDB) Search(filters *dbs.Filters, collection_name string) (*mongo.C f := dbs.GetBson(filters) + opts.SetSkip(offset) // OFFSET + opts.SetLimit(limit) // LIMIT + MngoCtx, cancel = context.WithTimeout(context.Background(), 5*time.Second) // defer cancel() if cursor, err := targetDBCollection.Find( @@ -329,7 +331,8 @@ func (m *MongoDB) LoadFilter(filter map[string]interface{}, collection_name stri return res, 200, nil } -func (m *MongoDB) LoadAll(collection_name string) (*mongo.Cursor, int, error) { +func (m *MongoDB) LoadAll(collection_name string, offset int64, limit int64) (*mongo.Cursor, int, error) { + if err := m.createClient(mngoConfig.GetUrl(), false); err != nil { return nil, 503, err } @@ -337,8 +340,10 @@ func (m *MongoDB) LoadAll(collection_name string) (*mongo.Cursor, int, error) { MngoCtx, cancel = context.WithTimeout(context.Background(), 5*time.Second) //defer cancel() - - res, err := targetDBCollection.Find(MngoCtx, bson.D{}) + findOptions := options.Find() + findOptions.SetSkip(offset) // OFFSET + findOptions.SetLimit(limit) // LIMIT + res, err := targetDBCollection.Find(MngoCtx, bson.D{}, findOptions) if err != nil { // m.Logger.Error().Msg("Couldn't find any resources. Error : " + err.Error()) return nil, 404, err diff --git a/entrypoint.go b/entrypoint.go index 5537f16..d6322e3 100644 --- a/entrypoint.go +++ b/entrypoint.go @@ -358,7 +358,7 @@ func (r *Request) PaymentTunnel(o *order.Order, scheduler *workflow_execution.Wo * @param c ...*tools.HTTPCaller * @return data LibDataShallow */ -func (r *Request) Search(filters *dbs.Filters, word string, isDraft bool) (data LibDataShallow) { +func (r *Request) Search(filters *dbs.Filters, word string, isDraft bool, offset int64, limit int64) (data LibDataShallow) { defer func() { // recover the panic if r := recover(); r != nil { tools.UncatchedError = append(tools.UncatchedError, errors.New("Panic recovered in Search : "+fmt.Sprintf("%v", r))) @@ -371,7 +371,7 @@ func (r *Request) Search(filters *dbs.Filters, word string, isDraft bool) (data PeerID: r.PeerID, Groups: r.Groups, Admin: r.admin, - }).Search(filters, word, isDraft) + }).Search(filters, word, isDraft, offset, limit) if err != nil { data = LibDataShallow{Data: d, Code: code, Err: err.Error()} return @@ -386,7 +386,7 @@ func (r *Request) Search(filters *dbs.Filters, word string, isDraft bool) (data * @param c ...*tools.HTTPCaller * @return data LibDataShallow */ -func (r *Request) LoadAll(isDraft bool) (data LibDataShallow) { +func (r *Request) LoadAll(isDraft bool, offset int64, limit int64) (data LibDataShallow) { defer func() { // recover the panic if r := recover(); r != nil { tools.UncatchedError = append(tools.UncatchedError, errors.New("Panic recovered in LoadAll : "+fmt.Sprintf("%v", r)+" - "+string(debug.Stack()))) @@ -399,7 +399,7 @@ func (r *Request) LoadAll(isDraft bool) (data LibDataShallow) { PeerID: r.PeerID, Groups: r.Groups, Admin: r.admin, - }).LoadAll(isDraft) + }).LoadAll(isDraft, offset, limit) if err != nil { data = LibDataShallow{Data: d, Code: code, Err: err.Error()} return @@ -729,7 +729,7 @@ func InitNATSDecentralizedEmitter(authorizedDT ...tools.DataType) { return // don't trust anyone... only friends and foes are privilege } access := NewRequestAdmin(LibDataEnum(resp.Datatype), nil) - if data := access.Search(nil, fmt.Sprintf("%v", p[resp.SearchAttr]), false); len(data.Data) > 0 { + if data := access.Search(nil, fmt.Sprintf("%v", p[resp.SearchAttr]), false, 0, 1); len(data.Data) > 0 { delete(p, "id") access.UpdateOne(p, data.Data[0].GetID()) } else { @@ -748,8 +748,8 @@ func InitNATSDecentralizedEmitter(authorizedDT ...tools.DataType) { access := NewRequestAdmin(LibDataEnum(resp.Datatype), nil) err := json.Unmarshal(resp.Payload, &p) if err == nil { - if data := access.Search(nil, fmt.Sprintf("%v", p[resp.SearchAttr]), false); len(data.Data) > 0 { - access.DeleteOne(fmt.Sprintf("%v", p[resp.SearchAttr])) + if data := access.Search(nil, fmt.Sprintf("%v", p[resp.SearchAttr]), false, 0, 1); len(data.Data) > 0 { + access.DeleteOne(data.Data[0].GetID()) } } diff --git a/models/bill/bill.go b/models/bill/bill.go index 0442cd5..cf25c40 100644 --- a/models/bill/bill.go +++ b/models/bill/bill.go @@ -221,7 +221,7 @@ func (d *PeerItemOrder) GetPriceHT(request *tools.APIRequest) (float64, error) { And: map[string][]dbs.Filter{ "resource_id": {{Operator: dbs.EQUAL.String(), Value: priced.GetID()}}, }, - }, "", d.Purchase.IsDraft) + }, "", d.Purchase.IsDraft, 0, 10000) if code == 200 && len(search) > 0 { for _, s := range search { if s.(*purchase_resource.PurchaseResource).EndDate == nil || time.Now().UTC().After(*s.(*purchase_resource.PurchaseResource).EndDate) { diff --git a/models/booking/planner/planner.go b/models/booking/planner/planner.go index c6a2e0c..47adb5c 100644 --- a/models/booking/planner/planner.go +++ b/models/booking/planner/planner.go @@ -74,7 +74,7 @@ func generate(request *tools.APIRequest, shallow bool) (*Planner, error) { And: map[string][]dbs.Filter{ "expected_start_date": {{Operator: dbs.GTE.String(), Value: time.Now().UTC()}}, }, - }, "*", false) + }, "*", false, 0, 10000) if code != 200 || err != nil { return nil, err } @@ -82,7 +82,7 @@ func generate(request *tools.APIRequest, shallow bool) (*Planner, error) { And: map[string][]dbs.Filter{ "expected_start_date": {{Operator: dbs.GTE.String(), Value: time.Now().UTC()}}, }, - }, "*", true) + }, "*", true, 0, 10000) bookings := append(confirmed, drafts...) p := &Planner{ diff --git a/models/collaborative_area/collaborative_area_mongo_accessor.go b/models/collaborative_area/collaborative_area_mongo_accessor.go index 167be34..5c53033 100644 --- a/models/collaborative_area/collaborative_area_mongo_accessor.go +++ b/models/collaborative_area/collaborative_area_mongo_accessor.go @@ -91,7 +91,7 @@ func filterEnrich[T utils.ShallowDBObject](arr []string, isDrafted bool, a utils Or: map[string][]dbs.Filter{ "abstractobject.id": {{Operator: dbs.IN.String(), Value: arr}}, }, - }, "", isDrafted) + }, "", isDrafted, 0, int64(len(arr))) if code == 200 { for _, r := range res { new = append(new, r.(T)) diff --git a/models/resources/native_tools.go b/models/resources/native_tools.go index 6f18ba8..1ffc979 100644 --- a/models/resources/native_tools.go +++ b/models/resources/native_tools.go @@ -61,7 +61,7 @@ func InitNative() { for _, kind := range []native_tools.NativeToolsEnum{native_tools.WORKFLOW_EVENT} { newNative := &NativeTool{} access := newNative.GetAccessor(&tools.APIRequest{Admin: true}) - l, _, err := access.Search(nil, kind.String(), false) + l, _, err := access.Search(nil, kind.String(), false, 0, 10) if err != nil || len(l) == 0 { newNative.Name = kind.String() newNative.Kind = int(kind) diff --git a/models/resources/resource_accessor.go b/models/resources/resource_accessor.go index 6daa0d2..4452867 100755 --- a/models/resources/resource_accessor.go +++ b/models/resources/resource_accessor.go @@ -80,22 +80,22 @@ func (dca *ResourceMongoAccessor[T]) CopyOne(data utils.DBObject) (utils.DBObjec return dca.StoreOne(data) } -func (wfa *ResourceMongoAccessor[T]) LoadAll(isDraft bool) ([]utils.ShallowDBObject, int, error) { - return utils.GenericLoadAll[T](wfa.GetExec(isDraft), isDraft, wfa) +func (wfa *ResourceMongoAccessor[T]) LoadAll(isDraft bool, offset int64, limit int64) ([]utils.ShallowDBObject, int, error) { + return utils.GenericLoadAll[T](wfa.GetExec(isDraft), isDraft, wfa, offset, limit) } -func (wfa *ResourceMongoAccessor[T]) Search(filters *dbs.Filters, search string, isDraft bool) ([]utils.ShallowDBObject, int, error) { +func (wfa *ResourceMongoAccessor[T]) Search(filters *dbs.Filters, search string, isDraft bool, offset int64, limit int64) ([]utils.ShallowDBObject, int, error) { if filters == nil && search == "*" { return utils.GenericLoadAll[T](func(d utils.DBObject) utils.ShallowDBObject { d.(T).SetAllowedInstances(wfa.Request) return d - }, isDraft, wfa) + }, isDraft, wfa, offset, limit) } return utils.GenericSearch[T](filters, search, wfa.GetObjectFilters(search), func(d utils.DBObject) utils.ShallowDBObject { d.(T).SetAllowedInstances(wfa.Request) return d - }, isDraft, wfa) + }, isDraft, wfa, offset, limit) } func (a *ResourceMongoAccessor[T]) GetExec(isDraft bool) func(utils.DBObject) utils.ShallowDBObject { diff --git a/models/utils/abstracts.go b/models/utils/abstracts.go index d9603e1..bc71260 100755 --- a/models/utils/abstracts.go +++ b/models/utils/abstracts.go @@ -266,12 +266,12 @@ func (a *AbstractAccessor[T]) LoadOne(id string) (DBObject, int, error) { }, a) } -func (a *AbstractAccessor[T]) LoadAll(isDraft bool) ([]ShallowDBObject, int, error) { - return GenericLoadAll[T](a.GetExec(isDraft), isDraft, a) +func (a *AbstractAccessor[T]) LoadAll(isDraft bool, offset int64, limit int64) ([]ShallowDBObject, int, error) { + return GenericLoadAll[T](a.GetExec(isDraft), isDraft, a, offset, limit) } -func (a *AbstractAccessor[T]) Search(filters *dbs.Filters, search string, isDraft bool) ([]ShallowDBObject, int, error) { - return GenericSearch[T](filters, search, a.New().GetObjectFilters(search), a.GetExec(isDraft), isDraft, a) +func (a *AbstractAccessor[T]) Search(filters *dbs.Filters, search string, isDraft bool, offset int64, limit int64) ([]ShallowDBObject, int, error) { + return GenericSearch[T](filters, search, a.New().GetObjectFilters(search), a.GetExec(isDraft), isDraft, a, offset, limit) } func (a *AbstractAccessor[T]) GetExec(isDraft bool) func(DBObject) ShallowDBObject { diff --git a/models/utils/common.go b/models/utils/common.go index c467697..b93c7c5 100755 --- a/models/utils/common.go +++ b/models/utils/common.go @@ -51,7 +51,7 @@ func GenericStoreOne(data DBObject, a Accessor) (DBObject, int, error) { if a.ShouldVerifyAuth() && !data.VerifyAuth("store", a.GetRequest()) { return nil, 403, errors.New("you are not allowed to access : " + a.GetType().String()) } - if cursor, _, _ := a.Search(&f, "", data.IsDrafted()); len(cursor) > 0 { + if cursor, _, _ := a.Search(&f, "", data.IsDrafted(), 0, 10); len(cursor) > 0 { return nil, 409, errors.New(a.GetType().String() + " with name " + data.GetName() + " already exists") } err := validate.Struct(data) @@ -172,17 +172,27 @@ func genericLoadAll[T DBObject](res *mgb.Cursor, code int, err error, onlyDraft return objs, 200, nil } -func GenericLoadAll[T DBObject](f func(DBObject) ShallowDBObject, onlyDraft bool, wfa Accessor) ([]ShallowDBObject, int, error) { - res_mongo, code, err := mongo.MONGOService.LoadAll(wfa.GetType().String()) +func GenericLoadAll[T DBObject](f func(DBObject) ShallowDBObject, onlyDraft bool, wfa Accessor, opts ...int64) ([]ShallowDBObject, int, error) { + offset := int64(0) + limit := int64(0) + if len(opts) > 1 { + offset = opts[0] + } + res_mongo, code, err := mongo.MONGOService.LoadAll(wfa.GetType().String(), offset, limit) return genericLoadAll[T](res_mongo, code, err, onlyDraft, f, wfa) } func GenericSearch[T DBObject](filters *dbs.Filters, search string, defaultFilters *dbs.Filters, - f func(DBObject) ShallowDBObject, onlyDraft bool, wfa Accessor) ([]ShallowDBObject, int, error) { + f func(DBObject) ShallowDBObject, onlyDraft bool, wfa Accessor, opts ...int64) ([]ShallowDBObject, int, error) { if filters == nil && search != "" { filters = defaultFilters } - res_mongo, code, err := mongo.MONGOService.Search(filters, wfa.GetType().String()) + offset := int64(0) + limit := int64(0) + if len(opts) > 1 { + offset = opts[0] + } + res_mongo, code, err := mongo.MONGOService.Search(filters, wfa.GetType().String(), offset, limit) return genericLoadAll[T](res_mongo, code, err, onlyDraft, f, wfa) } @@ -202,7 +212,7 @@ func GetMySelf(wfa Accessor) (ShallowDBObject, error) { And: map[string][]dbs.Filter{ "relation": {{Operator: dbs.EQUAL.String(), Value: 1}}, }, - }, "", false) + }, "", false, 0, 1) if len(datas) > 0 && datas[0] != nil { return datas[0], nil } diff --git a/models/utils/interfaces.go b/models/utils/interfaces.go index 6c22382..3ae8962 100755 --- a/models/utils/interfaces.go +++ b/models/utils/interfaces.go @@ -55,8 +55,8 @@ type Accessor interface { DeleteOne(id string) (DBObject, int, error) CopyOne(data DBObject) (DBObject, int, error) StoreOne(data DBObject) (DBObject, int, error) - LoadAll(isDraft bool) ([]ShallowDBObject, int, error) + LoadAll(isDraft bool, offset int64, limit int64) ([]ShallowDBObject, int, error) UpdateOne(set map[string]interface{}, id string) (DBObject, int, error) - Search(filters *dbs.Filters, search string, isDraft bool) ([]ShallowDBObject, int, error) + Search(filters *dbs.Filters, search string, isDraft bool, offset int64, limit int64) ([]ShallowDBObject, int, error) GetExec(isDraft bool) func(DBObject) ShallowDBObject } diff --git a/models/workflow/workflow_mongo_accessor.go b/models/workflow/workflow_mongo_accessor.go index 46c2993..3618a26 100644 --- a/models/workflow/workflow_mongo_accessor.go +++ b/models/workflow/workflow_mongo_accessor.go @@ -150,7 +150,7 @@ func (a *workflowMongoAccessor) execute(workflow *Workflow, delete bool, active "abstractobject.name": {{Operator: dbs.LIKE.String(), Value: workflow.Name + "_workspace"}}, }, } - resource, _, err := a.workspaceAccessor.Search(filters, "", workflow.IsDraft) + resource, _, err := a.workspaceAccessor.Search(filters, "", workflow.IsDraft, 0, 10) if delete { // if delete is set to true, delete the workspace for _, r := range resource { a.workspaceAccessor.DeleteOne(r.GetID()) @@ -192,9 +192,9 @@ func (a *workflowMongoAccessor) LoadOne(id string) (utils.DBObject, int, error) }, a) } -func (a *workflowMongoAccessor) Search(filters *dbs.Filters, search string, isDraft bool) ([]utils.ShallowDBObject, int, error) { +func (a *workflowMongoAccessor) Search(filters *dbs.Filters, search string, isDraft bool, offset int64, limit int64) ([]utils.ShallowDBObject, int, error) { return utils.GenericSearch[*Workflow](filters, search, a.New().GetObjectFilters(search), - func(d utils.DBObject) utils.ShallowDBObject { return a.verifyResource(d) }, isDraft, a) + func(d utils.DBObject) utils.ShallowDBObject { return a.verifyResource(d) }, isDraft, a, offset, limit) } func (a *workflowMongoAccessor) verifyResource(obj utils.DBObject) utils.DBObject { diff --git a/models/workflow_execution/workflow_execution.go b/models/workflow_execution/workflow_execution.go index e273c2c..f7f8214 100755 --- a/models/workflow_execution/workflow_execution.go +++ b/models/workflow_execution/workflow_execution.go @@ -89,7 +89,7 @@ func (ws *WorkflowExecution) PurgeDraft(request *tools.APIRequest) error { {Operator: dbs.GTE.String(), Value: primitive.NewDateTimeFromTime(ws.ExecDate)}, }, }, - }, "", ws.IsDraft) + }, "", ws.IsDraft, 0, 10000) if code != 200 || err != nil { return err } diff --git a/models/workspace/workspace_mongo_accessor.go b/models/workspace/workspace_mongo_accessor.go index bd48230..4f93344 100644 --- a/models/workspace/workspace_mongo_accessor.go +++ b/models/workspace/workspace_mongo_accessor.go @@ -49,7 +49,7 @@ func (a *workspaceMongoAccessor) DeleteOne(id string) (utils.DBObject, int, erro // UpdateOne updates a workspace in the database, given its ID, it automatically share to peers if the workspace is shared func (a *workspaceMongoAccessor) UpdateOne(set map[string]interface{}, id string) (utils.DBObject, int, error) { - if set["active"] == true { // If the workspace is active, deactivate all the other workspaces + /*if set["active"] == true { // If the workspace is active, deactivate all the other workspaces res, _, err := a.LoadAll(true) if err == nil { for _, r := range res { @@ -59,7 +59,7 @@ func (a *workspaceMongoAccessor) UpdateOne(set map[string]interface{}, id string } } } - } + }*/ res, code, err := utils.GenericUpdateOne(set, id, a) if code == 200 && res != nil { a.share(res.(*Workspace), tools.PUT, a.GetCaller()) @@ -76,8 +76,8 @@ func (a *workspaceMongoAccessor) StoreOne(data utils.DBObject) (utils.DBObject, }, } // filters *dbs.Filters, word string, isDraft bool - res, _, err := a.Search(filters, "", true) // Search for the workspace - if err == nil && len(res) > 0 { // If the workspace already exists, return an error + res, _, err := a.Search(filters, "", true, 0, 10) // Search for the workspace + if err == nil && len(res) > 0 { // If the workspace already exists, return an error return nil, 409, errors.New("a workspace with the same name already exists") } // reset the resources @@ -87,24 +87,24 @@ func (a *workspaceMongoAccessor) StoreOne(data utils.DBObject) (utils.DBObject, } func (a *workspaceMongoAccessor) LoadOne(id string) (utils.DBObject, int, error) { - return utils.GenericLoadOne[*Workspace](id, a.New(), func(d utils.DBObject) (utils.DBObject, int, error) { + return utils.GenericLoadOne(id, a.New(), func(d utils.DBObject) (utils.DBObject, int, error) { d.(*Workspace).Fill(a.GetRequest()) return d, 200, nil }, a) } -func (a *workspaceMongoAccessor) LoadAll(isDraft bool) ([]utils.ShallowDBObject, int, error) { +func (a *workspaceMongoAccessor) LoadAll(isDraft bool, offset int64, limit int64) ([]utils.ShallowDBObject, int, error) { return utils.GenericLoadAll[*Workspace](func(d utils.DBObject) utils.ShallowDBObject { d.(*Workspace).Fill(a.GetRequest()) return d - }, isDraft, a) + }, isDraft, a, offset, limit) } -func (a *workspaceMongoAccessor) Search(filters *dbs.Filters, search string, isDraft bool) ([]utils.ShallowDBObject, int, error) { +func (a *workspaceMongoAccessor) Search(filters *dbs.Filters, search string, isDraft bool, offset int64, limit int64) ([]utils.ShallowDBObject, int, error) { return utils.GenericSearch[*Workspace](filters, search, (&Workspace{}).GetObjectFilters(search), func(d utils.DBObject) utils.ShallowDBObject { d.(*Workspace).Fill(a.GetRequest()) return d - }, isDraft, a) + }, isDraft, a, offset, limit) } /*