From 2eb74da9d2513be2b005a053faa7e5f80ea24e93 Mon Sep 17 00:00:00 2001 From: pb Date: Wed, 17 Jul 2024 18:02:30 +0200 Subject: [PATCH] packaged every model --- datacenter/datacenter.go | 39 +++++ datacenter/datacenter_mongo_accessor.go | 33 ++++ datacenter/datacenter_test.go | 47 ++++++ link/link.go | 7 + logs/logger.go | 40 +++++ mongo/mongo.go | 209 ++++++++++++++++++++++++ processing/processing.go | 32 ++++ processing/processing_mongo_accessor.go | 38 +++++ processing/processing_test.go | 48 ++++++ storage/storage.go | 24 +++ storage/storage_mongo_accessor.go | 38 +++++ storage/storage_test.go | 48 ++++++ workflow/workflow.go | 34 ++++ workflow/workflow_mongo_accessor.go | 34 ++++ workflow/workflow_schedule.go | 15 ++ workflow/workflow_test.go | 45 +++++ 16 files changed, 731 insertions(+) create mode 100644 datacenter/datacenter.go create mode 100644 datacenter/datacenter_mongo_accessor.go create mode 100644 datacenter/datacenter_test.go create mode 100644 link/link.go create mode 100644 logs/logger.go create mode 100644 mongo/mongo.go create mode 100644 processing/processing.go create mode 100644 processing/processing_mongo_accessor.go create mode 100644 processing/processing_test.go create mode 100644 storage/storage.go create mode 100644 storage/storage_mongo_accessor.go create mode 100644 storage/storage_test.go create mode 100644 workflow/workflow.go create mode 100644 workflow/workflow_mongo_accessor.go create mode 100644 workflow/workflow_schedule.go create mode 100644 workflow/workflow_test.go diff --git a/datacenter/datacenter.go b/datacenter/datacenter.go new file mode 100644 index 0000000..73ca4ad --- /dev/null +++ b/datacenter/datacenter.go @@ -0,0 +1,39 @@ +package datacenter + +import oclib "oc-lib" + +type Datacenter struct { + oclib.AbstractResource `json:"resource" required:"true"` + + Owner string `json:"owner" required:"true"` + BookingPrice int `json:"booking_price" required:"true"` + + CPU DatacenterCpuModel `json:"cpu,omitempty"` + RAM DatacenterMemoryModel `json:"ram,omitempty"` + GPU []DatacenterGpuModel `json:"gpu,omitempty"` +} + +type DatacenterCpuModel struct { + Cores uint `json:"cores,omitempty"` //TODO: validate + Architecture string `json:"architecture,omitempty"` //TOOD: enum + Shared bool `json:"shared,omitempty"` + MinimumMemory uint `json:"minimum_memory,omitempty"` + Platform string `json:"platform,omitempty"` +} + +type DatacenterMemoryModel struct { + Size uint `json:"size,omitempty" description:"Units in MB"` + Ecc bool `json:"ecc,omitempty"` +} + +type DatacenterGpuModel struct { + CudaCores uint `json:"cuda_cores,omitempty"` + Model string `json:"model,omitempty"` + Memory uint `json:"memory,omitempty" description:"Units in MB"` + TensorCores uint `json:"tensor_cores,omitempty"` +} + + +func (d *Datacenter) GetType() oclib.ResourceType{ + return oclib.DATACENTER +} \ No newline at end of file diff --git a/datacenter/datacenter_mongo_accessor.go b/datacenter/datacenter_mongo_accessor.go new file mode 100644 index 0000000..be99fc6 --- /dev/null +++ b/datacenter/datacenter_mongo_accessor.go @@ -0,0 +1,33 @@ +package datacenter + +import ( + logs "oc-lib/logs" + "oc-lib/mongo" +) + +type DatacenterMongoAccessor struct{} + +func (dca *DatacenterMongoAccessor) StoreOne(datacenter Datacenter) string { + id, err := mongo.StoreOne(datacenter, "datacenter") + if err != nil { + l := logs.CreateLogger("oclib", "") + l.Error().Msg("Could not store " + datacenter.Name + " to db. Error: " + err.Error()) + return "" + } + return id +} + +func (dca *DatacenterMongoAccessor) LoadOne(id string) Datacenter { + var datacenter Datacenter + + res_mongo, err := mongo.LoadOne(id, "datacenter") + if err != nil { + l := logs.CreateLogger("oclib", "") + l.Error().Msg("Could not retrieve " + id + " from db. Error: " + err.Error()) + return Datacenter{} + } + + res_mongo.Decode(&datacenter) + + return datacenter +} diff --git a/datacenter/datacenter_test.go b/datacenter/datacenter_test.go new file mode 100644 index 0000000..4397e1f --- /dev/null +++ b/datacenter/datacenter_test.go @@ -0,0 +1,47 @@ +package datacenter + +import ( + oclib "oc-lib" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStoreOneDatacenter(t *testing.T){ + dc := Datacenter{Owner: "toto", BookingPrice: 123, + AbstractResource: oclib.AbstractResource{ + Uuid: "123", + Name: "testDatacenter", + Description: "Lorem Ipsum", + Logo : "azerty.com", + Owner: "toto", + OwnerLogo: "totoLogo", + SourceUrl: "azerty.fr", + }, + } + + dcma := DatacenterMongoAccessor{} + id := dcma.StoreOne(dc) + + assert.NotEmpty(t, id) +} + +func TestLoadOneDatacenter(t *testing.T){ + dc := Datacenter{Owner: "toto", BookingPrice: 123, + AbstractResource: oclib.AbstractResource{ + Uuid: "123", + Name: "testDatacenter", + Description: "Lorem Ipsum", + Logo : "azerty.com", + Owner: "toto", + OwnerLogo: "totoLogo", + SourceUrl: "azerty.fr", + }, + } + + dcma := DatacenterMongoAccessor{} + id := dcma.StoreOne(dc) + new_dc := dcma.LoadOne(id) + + assert.Equal(t,dc, new_dc) +} \ No newline at end of file diff --git a/link/link.go b/link/link.go new file mode 100644 index 0000000..832c2c9 --- /dev/null +++ b/link/link.go @@ -0,0 +1,7 @@ +package link + +type Link struct { + Source string + Destination string +} + diff --git a/logs/logger.go b/logs/logger.go new file mode 100644 index 0000000..e81ba63 --- /dev/null +++ b/logs/logger.go @@ -0,0 +1,40 @@ +package logs + +import ( + "os" + "runtime" + "time" + + "github.com/rs/zerolog" +) + +var logger zerolog.Logger + +// logs.CreateLogger +// Create a new logger +// Parameters: +// - appname: string : the name of the application using oclib +// - url: string : the url of a loki logger, console log only if "" +// Returns: +// - zerolog.Logger : the logger that will log for the library and the app +func CreateLogger(appname string, url string) zerolog.Logger { + + if url != "" { + labels := map[string]string{ + "app": "app", + "code": "go", + "platform": runtime.GOOS, + } + + lokiWriter := NewLokiWriter(url, labels) + + consoleWriter := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC3339} + + multiWriter := zerolog.MultiLevelWriter(consoleWriter, lokiWriter) + + logger = zerolog.New(multiWriter).With().Timestamp().Logger() + } else { + logger = zerolog.New(os.Stdout).With().Timestamp().Logger() + } + return logger +} diff --git a/mongo/mongo.go b/mongo/mongo.go new file mode 100644 index 0000000..4424e51 --- /dev/null +++ b/mongo/mongo.go @@ -0,0 +1,209 @@ +package mongo + +import ( + "context" + "encoding/json" + "errors" + "os" + "time" + + "github.com/rs/zerolog" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + + "oc-lib/logs" +) + +var ( + mngoClient *mongo.Client + mngoDB *mongo.Database + MngoCtx context.Context + cancel context.CancelFunc + logger zerolog.Logger + existingCollections []string + + ResourceMap map[string]interface{} + +) + + + +func init() { + + // var baseConfig string + var err error + var conf map[string]string + var MongoURL string + var DBname string + + ResourceMap = make(map[string]interface{}) + + logger := logs.CreateLogger("oclib","") + + db_conf, err := os.ReadFile("tests/oclib_conf.json") + if err != nil { + logger.Fatal().Msg("Could not find configuration file") + } + json.Unmarshal(db_conf,&conf) + + if len(os.Getenv("DOCKER_ENVIRONMENT")) == 0 { + MongoURL = conf["DB_URL_LOCAL"] + } else { + MongoURL = conf["DB_URL_DOCKER"] + } + + DBname = conf["DCNAME"] + "-" + conf["DBPOINT"] + + + logger.Info().Msg("Connecting to" + MongoURL) + + MngoCtx, cancel = context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + createClient(MongoURL) + + logger.Info().Msg("Connecting mongo client to db " + DBname) + prepareDB(conf["DCNAME"],conf["DBPOINT"]) + + logger.Info().Msg("Database is READY") + +} + +func createClient(MongoURL string){ + + var err error + + // Allows us to use marshal and unmarshall with results of FindOne() and others + bsonOpts := &options.BSONOptions { + UseJSONStructTags: true, + NilSliceAsEmpty: true, + } + + clientOptions := options.Client().ApplyURI(MongoURL).SetBSONOptions(bsonOpts) + mngoClient, err = mongo.Connect(MngoCtx,clientOptions) + if err != nil { + logger.Fatal().Msg("Mongodb NewClient " + MongoURL + ":" + "err" ) + panic(err) + } + + + // Ping the primary + if mngoClient, err = mongo.Connect(MngoCtx, clientOptions); err != nil { + logger.Fatal().Msg("Mongodb connect " + MongoURL + ":" + "err" ) + panic(err) + } + + if err = mngoClient.Ping(MngoCtx, nil); err != nil { + logger.Fatal().Msg("Mongodb ping " + MongoURL + ":" + "err" ) + panic(err) + } + +} + +func prepareDB(dc_name string, db_point string) { + + var err error + DBname := dc_name + "-" + db_point + mngoDB = mngoClient.Database(DBname) + + list_collection := [...]string{"data","processing","storage","datacenter","workspace","schedule","workflow"} + + existingCollections, err = mngoDB.ListCollectionNames(MngoCtx, bson.D{}) + if err != nil { + logger.Fatal().Msg("Error contacting MongoDB\n" + err.Error()) + } + collectionMap := make(map[string]bool) + for _, name := range existingCollections { + collectionMap[name] = true + } + + // Only do the collection definition process if it doesn't already exists + // we add the collection to the collection map from mongo/mongo_utils to provide faster access to the collection + + for _, collection_name := range(list_collection){ + new_collection := mngoDB.Collection(collection_name) + if _, exists := collectionMap[collection_name]; !exists { + createCollection(collection_name, new_collection) + } else{ + CollectionMap[collection_name] = new_collection + } + + } + +} + +// Creates the collection with index specified in mongo/mongo_collections +// or use the basic collection creation function +func createCollection(collection_name string, new_collection *mongo.Collection) { + + var err error + + CollectionMap[collection_name] = new_collection + + _, exists := IndexesMap[collection_name]; + if exists{ + if _, err = new_collection.Indexes().CreateMany(MngoCtx, IndexesMap[collection_name]); err != nil { + var cmdErr mongo.CommandError + if errors.As(err, &cmdErr) && cmdErr.Code != 85 { + logger.Fatal().Msg("Error creating indexes for " + collection_name + " collection : \n" + err.Error()) + panic(err) + } else if !errors.As(err, &cmdErr) { + logger.Fatal().Msg("Unexpected error: " + err.Error()) + panic(err) + } + } + } else { + mngoDB.CreateCollection(MngoCtx, collection_name) + } + +} + +func StoreOne(obj interface{}, collection_name string) (string, error) { + targetDBCollection := CollectionMap[collection_name] + + MngoCtx, cancel = context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + result, err := targetDBCollection.InsertOne(MngoCtx,obj) + if err != nil { + logger.Error().Msg("Couldn't insert resource: " + err.Error()) + return "", err + } + + return result.InsertedID.(primitive.ObjectID).Hex(), nil +} + +func LoadOne(id string, collection_name string) ( res *mongo.SingleResult , err error){ + + // new_obj := ResourceMap[collection_name] + // var doc bson.D + + filter := bson.M{"_id": GetObjIDFromString(id)} + targetDBCollection := CollectionMap[collection_name] + + MngoCtx, cancel = context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + res = targetDBCollection.FindOne(MngoCtx, filter) + + if res.Err() != nil { + logger.Error().Msg("Couldn't find resource " + id + ". Error : " + res.Err().Error()) + err = res.Err() + return nil, err + } + + return res, nil + +} + +func GetObjIDFromString(id string) interface{} { + objectID, err := primitive.ObjectIDFromHex(id) + if err == nil { + return objectID + } + + return id +} + diff --git a/processing/processing.go b/processing/processing.go new file mode 100644 index 0000000..f935aa2 --- /dev/null +++ b/processing/processing.go @@ -0,0 +1,32 @@ +package processing + +import oclib "oc-lib" + +type Processing struct { + oclib.AbstractResource `json:"resource" required:"true"` + Container string `json:"container,omitempty"` // We could create a specific class for container, that could check if the name exists/is available + Repository string `json:"repository,omitempty"` // Indicate where to find the container image => Could add a struct handling authentication to the repo + Command string `json:"command,omitempty"` + Arguments []string `json:"arguments,omitempty"` + Environment []map[string]string `json:"environment,omitempty"` // a key/value struct is what ressembles the most a NAME=VALUE struct + + ExecutionRequirements ExecutionRequirementsModel `json:"execution_requirements,omitempty"` + + Price uint `json:"price,omitempty"` + License string `json:"license,omitempty"` + +} + +type ExecutionRequirementsModel struct { + CPUs uint `json:"cp_us,omitempty"` + GPUs uint `json:"gp_us,omitempty"` + RAM uint `json:"ram,omitempty"` + + Parallel bool `json:"parallel,omitempty"` + ScalingModel uint `json:"scaling_model,omitempty"` + DiskIO string `json:"disk_io,omitempty"` +} + +func (p *Processing) GetType() oclib.ResourceType{ + return oclib.PROCESSING +} diff --git a/processing/processing_mongo_accessor.go b/processing/processing_mongo_accessor.go new file mode 100644 index 0000000..e41d4de --- /dev/null +++ b/processing/processing_mongo_accessor.go @@ -0,0 +1,38 @@ +package processing + +import ( + "oc-lib/logs" + "oc-lib/mongo" +) + + type ProcessingMongoAccessor struct{ + + } + + +func (pma *ProcessingMongoAccessor) StoreOne(processing Processing) string { + + id, err := mongo.StoreOne(processing,"processing") + if err != nil{ + l := logs.CreateLogger("oclib","") + l.Error().Msg("Could not store " + processing.Name + " to db. Error: " + err.Error()) + return "" + } + return id +} + +func (pma *ProcessingMongoAccessor) LoadOne(id string) Processing { + + var processing Processing + + res_mongo, err := mongo.LoadOne(id,"processing") + if err != nil{ + l := logs.CreateLogger("oclib","") + l.Error().Msg("Could not retrieve " + id + " from db. Error: " + err.Error()) + return Processing{} + } + + res_mongo.Decode(&processing) + + return processing +} \ No newline at end of file diff --git a/processing/processing_test.go b/processing/processing_test.go new file mode 100644 index 0000000..483fc05 --- /dev/null +++ b/processing/processing_test.go @@ -0,0 +1,48 @@ +package processing + +import ( + oclib "oc-lib" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStoreOneProcessing(t *testing.T){ + p := Processing{ Container: "totoCont", + AbstractResource: oclib.AbstractResource{ + Uuid: "123", + Name: "testData", + Description: "Lorem Ipsum", + Logo : "azerty.com", + Owner: "toto", + OwnerLogo: "totoLogo", + SourceUrl: "azerty.fr", + }, + } + + sma := ProcessingMongoAccessor{} + id := sma.StoreOne(p) + + assert.NotEmpty(t, id) +} + +func TestLoadOneProcessing(t *testing.T){ + p := Processing{ Container: "totoCont", + AbstractResource: oclib.AbstractResource{ + Uuid: "123", + Name: "testData", + Description: "Lorem Ipsum", + Logo : "azerty.com", + Owner: "toto", + OwnerLogo: "totoLogo", + SourceUrl: "azerty.fr", + }, + } + + + sma := ProcessingMongoAccessor{} + id := sma.StoreOne(p) + new_s := sma.LoadOne(id) + + assert.Equal(t,p, new_s) +} \ No newline at end of file diff --git a/storage/storage.go b/storage/storage.go new file mode 100644 index 0000000..c06a8c7 --- /dev/null +++ b/storage/storage.go @@ -0,0 +1,24 @@ +package storage + +import oclib "oc-lib" + +type URL struct { + Protocol string `json:"protocol"` + Path string `json:"path"` +} + +type Storage struct { + oclib.AbstractResource `json:"resource" required:"true" bson:"resource"` + + Capacity uint `json:"capacity,omitempty"` + Url URL `json:"url,omitempty"` // Will allow to select between several protocols + + Encryption bool `json:"encryption,omitempty"` + Redundancy string `json:"redundancy,omitempty"` + Throughput string `json:"throughput,omitempty"` + BookingPrice uint `json:"booking_price,omitempty"` +} + +func (s *Storage) GetType() oclib.ResourceType { + return oclib.STORAGE +} diff --git a/storage/storage_mongo_accessor.go b/storage/storage_mongo_accessor.go new file mode 100644 index 0000000..adf954f --- /dev/null +++ b/storage/storage_mongo_accessor.go @@ -0,0 +1,38 @@ +package storage + +import ( + "oc-lib/logs" + "oc-lib/mongo" +) + + type StorageMongoAccessor struct{ + + } + + +func (schedulema *StorageMongoAccessor) StoreOne(storage Storage) string { + + id, err := mongo.StoreOne(storage,"storage") + if err != nil{ + l := logs.CreateLogger("oclib","") + l.Error().Msg("Could not store " + storage.Name + " to db. Error: " + err.Error()) + return "" + } + return id +} + +func (schedulema *StorageMongoAccessor) LoadOne(id string) Storage { + + var storage Storage + + res_mongo, err := mongo.LoadOne(id,"storage") + if err != nil{ + l := logs.CreateLogger("oclib","") + l.Error().Msg("Could not retrieve " + id + " from db. Error: " + err.Error()) + return Storage{} + } + + res_mongo.Decode(&storage) + + return storage +} \ No newline at end of file diff --git a/storage/storage_test.go b/storage/storage_test.go new file mode 100644 index 0000000..1429dce --- /dev/null +++ b/storage/storage_test.go @@ -0,0 +1,48 @@ +package storage + +import ( + oclib "oc-lib" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStoreOneStorage(t *testing.T){ + s := Storage{ Capacity: 123, Url: URL{Protocol: "http",Path: "azerty.fr"} , + AbstractResource: oclib.AbstractResource{ + Uuid: "123", + Name: "testData", + Description: "Lorem Ipsum", + Logo : "azerty.com", + Owner: "toto", + OwnerLogo: "totoLogo", + SourceUrl: "azerty.fr", + }, + } + + sma := StorageMongoAccessor{} + id := sma.StoreOne(s) + + assert.NotEmpty(t, id) +} + +func TestLoadOneStorage(t *testing.T){ + s := Storage{ Capacity: 123, Url: URL{Protocol: "http",Path: "azerty.fr"} , + AbstractResource: oclib.AbstractResource{ + Uuid: "123", + Name: "testData", + Description: "Lorem Ipsum", + Logo : "azerty.com", + Owner: "toto", + OwnerLogo: "totoLogo", + SourceUrl: "azerty.fr", + }, + } + + + sma := StorageMongoAccessor{} + id := sma.StoreOne(s) + new_s := sma.LoadOne(id) + + assert.Equal(t,s, new_s) +} \ No newline at end of file diff --git a/workflow/workflow.go b/workflow/workflow.go new file mode 100644 index 0000000..3002156 --- /dev/null +++ b/workflow/workflow.go @@ -0,0 +1,34 @@ +package oclib + +import ( + oclib "oc-lib" + "oc-lib/data" + "oc-lib/datacenter" + "oc-lib/link" + "oc-lib/processing" + "oc-lib/storage" +) + +type Workflow struct { + oclib.AbstractResource `json:"abstract_resource" required:"true"` + Datas map[string]data.Data `json:"datas,omitempty"` + Storages map[string]storage.Storage `json:"storages,omitempty"` + Processing map[string]processing.Processing `json:"processing,omitempty"` + Datacenters map[string]datacenter.Datacenter `json:"datacenters,omitempty"` + Links map[string]link.Link `json:"links,omitempty"` + + Schedule WorkflowSchedule `json:"schedule,omitempty"` +} + +func (w *Workflow) isDCLink(link link.Link) bool { + if _, exists := w.Datacenters[link.Destination]; exists { + return true + } else if _, exists := w.Datacenters[link.Source]; exists { + return true + } + + return false +} + + + diff --git a/workflow/workflow_mongo_accessor.go b/workflow/workflow_mongo_accessor.go new file mode 100644 index 0000000..13a284b --- /dev/null +++ b/workflow/workflow_mongo_accessor.go @@ -0,0 +1,34 @@ +package oclib + +import ( + "oc-lib/logs" + "oc-lib/mongo" +) + + +type WorkflowMongoAccessor struct{} + +func (wfa *WorkflowMongoAccessor) StoreOne(workflow Workflow) string { + id, err := mongo.StoreOne(workflow, "workflow") + if err != nil { + l := logs.CreateLogger("oclib", "") + l.Error().Msg("Could not store " + workflow.Name + " to db. Error: " + err.Error()) + return "" + } + return id +} + +func (wfa *WorkflowMongoAccessor) LoadOne(id string) Workflow { + var workflow Workflow + + res_mongo, err := mongo.LoadOne(id, "workflow") + if err != nil { + l := logs.CreateLogger("oclib", "") + l.Error().Msg("Could not retrieve " + id + " from db. Error: " + err.Error()) + return Workflow{} + } + + res_mongo.Decode(&workflow) + + return workflow +} \ No newline at end of file diff --git a/workflow/workflow_schedule.go b/workflow/workflow_schedule.go new file mode 100644 index 0000000..7d5d5a1 --- /dev/null +++ b/workflow/workflow_schedule.go @@ -0,0 +1,15 @@ +package oclib + +import "time" + +type WorkflowSchedule struct { + Id string `json:"id"` + Start time.Time + End time.Time + Cron string +} + +func (ws *WorkflowSchedule) GetAllDates() (timetable []time.Time){ + // Return all the execution time generated by the Cron + return +} \ No newline at end of file diff --git a/workflow/workflow_test.go b/workflow/workflow_test.go new file mode 100644 index 0000000..90d16cd --- /dev/null +++ b/workflow/workflow_test.go @@ -0,0 +1,45 @@ +package oclib + +import ( + oclib "oc-lib" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStoreOneWorkflow(t *testing.T){ + w := Workflow{AbstractResource: oclib.AbstractResource{ + Uuid: "123", + Name: "testWorkflow", + Description: "Lorem Ipsum", + Logo : "azerty.com", + Owner: "toto", + OwnerLogo: "totoLogo", + SourceUrl: "azerty.fr", + }, + } + + wma := WorkflowMongoAccessor{} + id := wma.StoreOne(w) + + assert.NotEmpty(t, id) +} + +func TestLoadOneWorkflow(t *testing.T){ + w := Workflow{AbstractResource: oclib.AbstractResource{ + Uuid: "123", + Name: "testWorkflow", + Description: "Lorem Ipsum", + Logo : "azerty.com", + Owner: "toto", + OwnerLogo: "totoLogo", + SourceUrl: "azerty.fr", + }, + } + + wma := WorkflowMongoAccessor{} + id := wma.StoreOne(w) + new_w := wma.LoadOne(id) + + assert.Equal(t,w, new_w) +} \ No newline at end of file