diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2a94754 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +out/* \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..9c49fc7 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,26 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch test function", + "type": "go", + "request": "launch", + "mode": "test", + "program": "${workspaceFolder}", + "args": [ + "-test.run", + "TestMongoInit" + ] + }, + { + "name": "Launch Package", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${fileDirname}" + } + ] +} \ No newline at end of file diff --git a/conf.go b/conf.go new file mode 100644 index 0000000..b2cc5e4 --- /dev/null +++ b/conf.go @@ -0,0 +1,26 @@ +package oclib + +import "sync" + +// =================================================== +// This class has to be updated everytime +// a new configuration variable is defined +// in a componant that imports oc-lib +// =================================================== + +type Config struct { + MongoURL string + DCNAME string + DBPOINT string + +} + +var instance *Config +var once sync.Once + +func GetConfig() *Config { + once.Do(func() { + instance = &Config{} + }) + return instance +} diff --git a/data.go b/data.go index ad54c3d..92941ef 100644 --- a/data.go +++ b/data.go @@ -1,5 +1,16 @@ package oclib type Data struct { - Resource + AbstractResource + + Protocols []string `json:"protocol"` //TODO Enum type + DataType string `json:"datatype"` + Example string `json:"example" required:"true" validate:"required" description:"base64 encoded data"` + } + + +func (d *Data) GetType() ResourceType{ + return DATA +} + diff --git a/data_test.go b/data_test.go new file mode 100644 index 0000000..163d7dc --- /dev/null +++ b/data_test.go @@ -0,0 +1,9 @@ +package oclib + +import ( + "testing" +) + +func TestPostOneData(t *testing.T){ + +} \ No newline at end of file diff --git a/datacenter.go b/datacenter.go index 692632c..a0dec2b 100644 --- a/datacenter.go +++ b/datacenter.go @@ -1 +1,39 @@ package oclib + +type Datacenter struct { + Resource + ID string `json:"ID", required: "true"` + Name string `json:"Name", required: "true" ` + + Owner string `json:"owner" ` + BookingPrice int `json:"bookingPrice" ` + + CPU DatacenterCpuModel `json:"cpu" required:"true"` + RAM DatacenterMemoryModel `json:"ram" required:"true"` + GPU []DatacenterGpuModel `json:"gpu" required:"true"` +} + +type DatacenterCpuModel struct { + Cores uint `json:"cores" required:"true"` //TODO: validate + Architecture string `json:"architecture"` //TOOD: enum + Shared bool `json:"shared"` + MinimumMemory uint `json:"minimum_memory"` + Platform string `json:"platform"` +} + +type DatacenterMemoryModel struct { + Size uint `json:"size" description:"Units in MB"` + Ecc bool `json:"ecc"` +} + +type DatacenterGpuModel struct { + CudaCores uint `json:"cuda_cores"` + Model string `json:"model"` + Memory uint `json:"memory" description:"Units in MB"` + TensorCores uint `json:"tensor_cores"` +} + + +func (d *Datacenter) GetType() ResourceType{ + return DATACENTER +} \ No newline at end of file diff --git a/doc/model.puml b/doc/model.puml index f73439e..a7bd83b 100644 --- a/doc/model.puml +++ b/doc/model.puml @@ -2,7 +2,7 @@ abstract Ressource { - +id: int + +UUID: int +name: string +icon: string +description: string @@ -11,12 +11,12 @@ abstract Ressource { } class Data { - +id: int + +UUID: int +name: string } class Processing { - +id: int + +UUID: int +name: string +container: string +command: int @@ -25,83 +25,83 @@ class Processing { } class Storage { - +id: int + +UUID: int +name: string +url: string +capacity: int } class Datacenter { - +id: int + +UUID: int +name: string } class Workflow { - +id: int + +UUID: int +name: string } class ResourceSet { - +id: int + +UUID: int +name: string +ressources: Ressource[] } class WorkflowSchedule { - +id: int + +UUID: int +start: date +end: date +cron : string } class Graph { - +id: int - +ressources: map[GraphicElement.ID]Ressource + +UUID: int + +ressources: map[GraphicElement.UUID]Ressource +links: Link[] } class Link { - +id: int - +source: GraphicElement.ID - +target: GraphicElement.ID + +UUID: int + +source: GraphicElement.UUID + +target: GraphicElement.UUID +graphic: GraphicLink } class GraphicLink { - +id: int + +UUID: int +startXY: coord +endXY: coord +style: string } class GraphicElement { - +id: int + +UUID: int +style: string +xy: coord } class Calendar { - +id: int + +UUID: int +name: string +workflows: Workflow[] +owner: string } class UserWorkflows { - +id: int + +UUID: int +user: string +workflows: Workflow[] } class DatacenterWorkflows { - +id: int + +UUID: int +datacenter: Datacenter +workflows: Workflow[] } class Graph { - +id: int + +UUID: int +graph: Graph +workflows: Workflow[] } @@ -121,7 +121,7 @@ Ressource <|-- Workflow ResourceSet "1" o-- "0..*" Ressource Workflow "1" o-- "0..*" ResourceSet -Workflow "1" o-- "0..*" WorkflowSchedule +Workflow "1" o-- "0..1" WorkflowSchedule Workflow "1" o-- "0..*" Graph Graph "1" o-- "0..*" Resources diff --git a/doc/oclib.png b/doc/oclib.png index d554429..bbaad5e 100644 Binary files a/doc/oclib.png and b/doc/oclib.png differ diff --git a/graphic_element.go b/graphic_element.go new file mode 100644 index 0000000..b99325e --- /dev/null +++ b/graphic_element.go @@ -0,0 +1,12 @@ +package oclib + +type Coordinate struct { + x int + y int +} + +type GraphicElement struct{ + ID string `json:"ID" required:"true"` + style string `json:"style" required:"true"` + xy Coordinate `json:"xy" required:"true"` +} \ No newline at end of file diff --git a/link.go b/link.go new file mode 100644 index 0000000..7cd12b9 --- /dev/null +++ b/link.go @@ -0,0 +1,7 @@ +package oclib + +type Link struct { + Source string + Destination string +} + diff --git a/mongo.go b/mongo.go new file mode 100644 index 0000000..c558633 --- /dev/null +++ b/mongo.go @@ -0,0 +1,187 @@ +package oclib + +import ( + "context" + "encoding/json" + "errors" + "os" + "time" + + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/x/bsonx" +) + +var ( + mngoClient *mongo.Client + mngoDB *mongo.Database + MngoCtx context.Context + + MngoCollData *mongo.Collection + MngoCollComputing *mongo.Collection + MngoCollStorage *mongo.Collection + MngoCollDatacenter *mongo.Collection + MngoCollWorkspace *mongo.Collection + MngoCollSchedule *mongo.Collection +) + +// Trying to get vscode to display this +func MongoInit() { + + // var baseConfig string + var err error + var cancel context.CancelFunc + var conf map[string]string + var MongoURL string + var DBname string + + db_conf, err := os.ReadFile("oclib_conf.json") + if err != nil { + logger.Fatal().Msg("Could not find configuration file") + } + + json.Unmarshal(db_conf,&conf) + + + if len(os.Getenv("DOCKER_ENVIRONMENT")) == 0 { + MongoURL = conf["DB_URL_LOCAL"] + } else { + MongoURL = conf["DB_URL_DOCKER"] + } + + DBname = conf["DCNAME"] + "-" + conf["DBPOINT"] + + // MongoURL, err := beego.AppConfig.String(baseConfig + "::url") + // if err != nil { + // logger.Critical().Msg("MongoDB URI error: %v", err) + // panic(err) + // } + + logger.Info().Msg("Connecting to" + MongoURL) + MngoCtx, cancel = context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + createClient(MongoURL) + + logger.Info().Msg("Connecting mongo client to db " + DBname) + prepareDB(conf["DCNAME"],conf["DBPOINT"]) + + logger.Info().Msg("Database is READY") + +} + +// TODO : Soit retourner MngoCtx, soit comprendre comment passer sa référence et la mettre à jour +func createClient(MongoURL string){ + + var err error + + clientOptions := options.Client().ApplyURI(MongoURL) + + mngoClient, _ = mongo.NewClient(options.Client().ApplyURI(MongoURL)) + + if err = mngoClient.Connect(MngoCtx); err != nil { + logger.Fatal().Msg("Mongodb NewClient " + MongoURL + ":" + "err" ) + panic(err) + } + + + // Ping the primary + if mngoClient, err = mongo.Connect(MngoCtx, clientOptions); err != nil { + logger.Fatal().Msg("Mongodb connect " + MongoURL + ":" + "err" ) + panic(err) + } + + if err = mngoClient.Ping(MngoCtx, nil); err != nil { + logger.Fatal().Msg("Mongodb ping " + MongoURL + ":" + "err" ) + panic(err) + } + +} + +func prepareDB(dc_name string, db_point string) { + + // var err error + DBname := dc_name + "-" + db_point + + mngoDB = mngoClient.Database(DBname) + + MngoCollData = mngoDB.Collection("Data") + MngoCollComputing = mngoDB.Collection("Computing") + MngoCollStorage = mngoDB.Collection("Storage") + MngoCollDatacenter = mngoDB.Collection("Datacenter") + MngoCollWorkspace = mngoDB.Collection("Workspace") + MngoCollSchedule = mngoDB.Collection("Schedule") + + list_collection := [...]string{"Data","Computing","Storage","Datacenter","Workspace","Schedule"} + + for _, collection_name := range(list_collection){ + new_collection := mngoDB.Collection(collection_name) + createCollection(new_collection) + } + + + + + + // if _, err = MngoCollComputing.Indexes().CreateMany(MngoCtx, []mongo.IndexModel{ + // { + // Keys: bsonx.Doc{ + // {Key: "description", Value: bsonx.String("text")}, + // {Key: "owner", Value: bsonx.String("text")}, + // {Key: "license", Value: bsonx.String("text")}, + // }, + // }, + // }); err != nil && err.(mongo.CommandError).Code != 85 { + // logger.Critical().Msg(err) + // panic(err) + // } + + // if _, err = MngoCollStorage.Indexes().CreateMany(MngoCtx, []mongo.IndexModel{ + // { + // Keys: bsonx.Doc{ + // {Key: "name", Value: bsonx.String("text")}, + // {Key: "description", Value: bsonx.String("text")}, + // }, + // }, + // }); err != nil && err.(mongo.CommandError).Code != 85 { + // logger.Critical().Msg(err) + // panic(err) + // } + + // if _, err = MngoCollDatacenter.Indexes().CreateMany(MngoCtx, []mongo.IndexModel{ + // { + // Keys: bsonx.Doc{ + // {Key: "name", Value: bsonx.String("text")}, + // {Key: "description", Value: bsonx.String("text")}, + // {Key: "owner", Value: bsonx.String("text")}, + // }, + // }, + // }); err != nil && err.(mongo.CommandError).Code != 85 { + // logger.Critical().Msg(err) + // panic(err) + // } +} + +func createCollection(new_collection *mongo.Collection) { + + var err error + + if _, err = new_collection.Indexes().CreateMany(MngoCtx, []mongo.IndexModel{ + { + Keys: bsonx.Doc{ + {Key: "description", Value: bsonx.String("text")}, + {Key: "example", Value: bsonx.String("text")}, + }, + }, + }); err != nil { + var cmdErr mongo.CommandError + if errors.As(err, &cmdErr) && cmdErr.Code != 85 { + logger.Fatal().Msg("It failed but I saw it: " + err.Error()) + panic(err) + } else if !errors.As(err, &cmdErr) { + logger.Fatal().Msg("Unexpected error: " + err.Error()) + panic(err) + } + } + +} \ No newline at end of file diff --git a/mongo_test.go b/mongo_test.go new file mode 100644 index 0000000..576318a --- /dev/null +++ b/mongo_test.go @@ -0,0 +1,10 @@ +package oclib + +import ( + "fmt" + "testing" +) +func TestMongoInit(T *testing.T){ + MongoInit() + fmt.Printf("It worked !") +} \ No newline at end of file diff --git a/oclib_conf.json b/oclib_conf.json new file mode 100644 index 0000000..fa909a7 --- /dev/null +++ b/oclib_conf.json @@ -0,0 +1,6 @@ +{ + "DB_URL_LOCAL" : "mongodb://127.0.0.1:27017", + "DB_URL_DOCKER": "mongodb://mongo:27017/", + "DBPOINT" : "oclib_tests", + "DCNAME" : "testDC" +} \ No newline at end of file diff --git a/processing.go b/processing.go new file mode 100644 index 0000000..3a65715 --- /dev/null +++ b/processing.go @@ -0,0 +1,30 @@ +package oclib + +type Processing struct { + Resource + Container string `json:"Container"` // We could create a specific class for container, that could check if the name exists/is available + Repository string `json:"Repository"` // Indicate where to find the container image => Could add a struct handling authentication to the repo + Command string `json:"Command"` + Arguments []string `json:"Arguments"` + Environment []map[string]string `json:"Environment"` // a key/value struct is what ressembles the most a NAME=VALUE struct + + ExecutionRequirements ExecutionRequirementsModel `json:"ExecutionRequirements"` + + Price uint `json:"Price"` + License string `json:"License"` + +} + +type ExecutionRequirementsModel struct { + CPUs uint `json:"cpus" required:"true"` + GPUs uint `json:"gpus" description:"Amount of GPUs needed"` + RAM uint `json:"ram" required:"true" description:"Units in MB"` + + Parallel bool `json:"parallel"` + ScalingModel uint `json:"scaling_model"` + DiskIO string `json:"disk_io"` +} + +func (p *Processing) GetType() ResourceType{ + return PROCESSING +} diff --git a/resource.go b/resource.go index 12eea5c..c8defbb 100644 --- a/resource.go +++ b/resource.go @@ -1,12 +1,94 @@ package oclib -type Resource struct { - Id string - Name string - ShortDescription string - Description string - Logo string - Owner string - OwnerLogo string - SourceUrl string +import ( + "go.mongodb.org/mongo-driver/bson/primitive" +) + +// AbstractResource is the struct containing all of the attributes commons to all ressources + +// Resource is the interface to be implemented by all classes inheriting from Resource to have the same behavior + +//http://www.inanzzz.com/index.php/post/wqbs/a-basic-usage-of-int-and-string-enum-types-in-golang + +type ResourceType int + +const ( + INVALID ResourceType = iota + DATA + PROCESSING + STORAGE + DATACENTER + WORKFLOW +) + +var extensions = [...]string{ + "INVALID", + "DATA", + "PROCESSING", + "STORAGE", + "DATACENTER", + "WORKFLOW", } + + +type Resource interface{ + GetType() ResourceType + getOneResourceByID() Resource +} + +type AbstractResource struct { + Uuid string `json:"Id" required:"true" ` + Name string `json:"Name" required:"true" ` + ShortDescription string + Description string + Logo string + Owner string + OwnerLogo string + SourceUrl string + + Graphic GraphicElement `json:"GraphicElement" ` +} + + +// func (r *AbstractResource) getOneResourceByID(id string, resType ResourceType){ + + +// targetDBCollection := r.GetType().MongoCollection() // Change the rType by the result of reflect +// var retObj interface{} + + + +// filter := bson.M{"_id": getObjIDFromString(id)} + +// res := targetDBCollection.FindOne(services.MngoCtx, filter) +// res.Decode(retObj) + +// if res.Err() != nil { +// logs.Warn("Couldn't find resource: " + res.Err().Error()) +// } + +// return retObj, res.Err() +// } + +func getObjIDFromString(id string) interface{} { + objectID, err := primitive.ObjectIDFromHex(id) + if err == nil { + return objectID + } + + return id +} + +func postToMongo(id string) { + +} + +func (r *AbstractResource) isLinked(){ + // Get the link collection in this workflow + // test if the current resource is a dest OR a source at least one + // (len(slice[r.ID == dest]) > 0 || len(slice[r.ID == 1]) > 1 ) +} + +// func (r *Resource) GetType() ResourceType { +// return INVALID +// } \ No newline at end of file diff --git a/resource_set.go b/resource_set.go index 692632c..12c28c0 100644 --- a/resource_set.go +++ b/resource_set.go @@ -1 +1,6 @@ package oclib + +// Resources' key must be the Resource' Uuid, to garanty the set-like structure +type WorkflowSet struct{ + Resources map[string]Resource +} \ No newline at end of file diff --git a/storage.go b/storage.go index 692632c..eb74204 100644 --- a/storage.go +++ b/storage.go @@ -1 +1,22 @@ package oclib + +type URL struct { + Protocol string `json:"protocol"` + Path string `json:"path"` +} + +type Storage struct { + Resource `json:"resource" required:"true"` + + Capacity uint `json:"capacity" required:"true"` + Url URL `json:"URL" ` // Will allow to select between several protocols + + Encryption bool `json:"encryption" ` + Redundancy string `json:"redundancy" ` + Throughput string `json:"throughput" ` + BookingPrice uint `json:"bookingPrice" ` +} + +func (s *Storage) GetType() ResourceType{ + return STORAGE +} diff --git a/workflow.go b/workflow.go new file mode 100644 index 0000000..43f2fed --- /dev/null +++ b/workflow.go @@ -0,0 +1,25 @@ +package oclib + +type Workflow struct{ + Resource + Datas map[string]Data + Storages map[string]Storage + Processing map[string]Processing + Datacenters map[string]Datacenter + Links map[string]Link + + Schedule WorkflowSchedule +} + +func (w *Workflow) isDCLink(link Link) bool { + if _, exists := w.Datacenters[link.Destination]; exists { + return true + } else if _, exists := w.Datacenters[link.Source]; exists { + return true + } + + return false +} + + + diff --git a/workflow_schedule.go b/workflow_schedule.go index 692632c..7d5d5a1 100644 --- a/workflow_schedule.go +++ b/workflow_schedule.go @@ -1 +1,15 @@ package oclib + +import "time" + +type WorkflowSchedule struct { + Id string `json:"id"` + Start time.Time + End time.Time + Cron string +} + +func (ws *WorkflowSchedule) GetAllDates() (timetable []time.Time){ + // Return all the execution time generated by the Cron + return +} \ No newline at end of file