packaged every model

datacenter/datacenter.go

@@ -0,0 +1,39 @@
+package datacenter
+
+import oclib "oc-lib"
+
+type Datacenter struct {
+	oclib.AbstractResource `json:"resource" required:"true"`
+
+	Owner        string `json:"owner" required:"true"`
+	BookingPrice int    `json:"booking_price" required:"true"`
+
+	CPU DatacenterCpuModel    `json:"cpu,omitempty"`
+	RAM DatacenterMemoryModel `json:"ram,omitempty"`
+	GPU []DatacenterGpuModel  `json:"gpu,omitempty"`
+}
+
+type DatacenterCpuModel struct {
+	Cores         uint   `json:"cores,omitempty"`        //TODO: validate
+	Architecture  string `json:"architecture,omitempty"` //TOOD: enum
+	Shared        bool   `json:"shared,omitempty"`
+	MinimumMemory uint   `json:"minimum_memory,omitempty"`
+	Platform      string `json:"platform,omitempty"`
+}
+
+type DatacenterMemoryModel struct {
+	Size uint `json:"size,omitempty" description:"Units in MB"`
+	Ecc  bool `json:"ecc,omitempty"`
+}
+
+type DatacenterGpuModel struct {
+	CudaCores   uint   `json:"cuda_cores,omitempty"`
+	Model       string `json:"model,omitempty"`
+	Memory      uint   `json:"memory,omitempty" description:"Units in MB"`
+	TensorCores uint   `json:"tensor_cores,omitempty"`
+}
+
+
+func (d *Datacenter) GetType() oclib.ResourceType{
+	return oclib.DATACENTER
+}

datacenter/datacenter_mongo_accessor.go

@@ -0,0 +1,33 @@
+package datacenter
+
+import (
+	logs "oc-lib/logs"
+	"oc-lib/mongo"
+)
+
+type DatacenterMongoAccessor struct{}
+
+func (dca *DatacenterMongoAccessor) StoreOne(datacenter Datacenter) string {
+    id, err := mongo.StoreOne(datacenter, "datacenter")
+    if err != nil {
+        l := logs.CreateLogger("oclib", "")
+        l.Error().Msg("Could not store " + datacenter.Name + " to db. Error: " + err.Error())
+        return ""
+    }
+    return id
+}
+
+func (dca *DatacenterMongoAccessor) LoadOne(id string) Datacenter {
+    var datacenter Datacenter
+
+    res_mongo, err := mongo.LoadOne(id, "datacenter")
+    if err != nil {
+        l := logs.CreateLogger("oclib", "")
+        l.Error().Msg("Could not retrieve " + id + " from db. Error: " + err.Error())
+        return Datacenter{}
+    }
+
+    res_mongo.Decode(&datacenter)
+
+    return datacenter
+}

datacenter/datacenter_test.go

@@ -0,0 +1,47 @@
+package datacenter
+
+import (
+	oclib "oc-lib"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStoreOneDatacenter(t *testing.T){
+	dc := Datacenter{Owner: "toto", BookingPrice: 123, 
+				AbstractResource: oclib.AbstractResource{
+					Uuid: "123",
+					Name: "testDatacenter",
+					Description: "Lorem Ipsum",
+					Logo : "azerty.com",
+					Owner: "toto",
+					OwnerLogo: "totoLogo",
+					SourceUrl: "azerty.fr",
+				},
+			}
+
+	dcma := DatacenterMongoAccessor{}
+	id := dcma.StoreOne(dc)
+
+	assert.NotEmpty(t, id)
+}
+
+func TestLoadOneDatacenter(t *testing.T){
+	dc := Datacenter{Owner: "toto", BookingPrice: 123, 
+				AbstractResource: oclib.AbstractResource{
+					Uuid: "123",
+					Name: "testDatacenter",
+					Description: "Lorem Ipsum",
+					Logo : "azerty.com",
+					Owner: "toto",
+					OwnerLogo: "totoLogo",
+					SourceUrl: "azerty.fr",
+				},
+			}
+	
+	dcma := DatacenterMongoAccessor{}
+	id := dcma.StoreOne(dc)
+	new_dc := dcma.LoadOne(id)
+
+	assert.Equal(t,dc, new_dc)
+}

link/link.go

@@ -0,0 +1,7 @@
+package link
+
+type Link struct {
+	Source		string
+	Destination	string
+}
+

logs/logger.go

@@ -0,0 +1,40 @@
+package logs
+
+import (
+	"os"
+	"runtime"
+	"time"
+
+	"github.com/rs/zerolog"
+)
+
+var logger zerolog.Logger
+
+// logs.CreateLogger
+// Create a new logger
+// Parameters:
+// - appname: string : the name of the application using oclib
+// - url: string : the url of a loki logger, console log only if ""
+// Returns:
+// - zerolog.Logger : the logger that will log for the library and the app
+func CreateLogger(appname string, url string) zerolog.Logger {
+
+	if url != "" {
+		labels := map[string]string{
+			"app":      "app",
+			"code":     "go",
+			"platform": runtime.GOOS,
+		}
+
+		lokiWriter := NewLokiWriter(url, labels)
+
+		consoleWriter := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC3339}
+
+		multiWriter := zerolog.MultiLevelWriter(consoleWriter, lokiWriter)
+
+		logger = zerolog.New(multiWriter).With().Timestamp().Logger()
+	} else {
+		logger = zerolog.New(os.Stdout).With().Timestamp().Logger()
+	}
+	return logger
+}

mongo/mongo.go

@@ -0,0 +1,209 @@
+package mongo
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"os"
+	"time"
+
+	"github.com/rs/zerolog"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+
+	"oc-lib/logs"
+)
+
+var (
+	mngoClient 	*mongo.Client
+	mngoDB     	*mongo.Database
+	MngoCtx    	context.Context
+	cancel 		context.CancelFunc
+	logger		zerolog.Logger
+	existingCollections	[]string
+
+	ResourceMap map[string]interface{}
+
+)
+
+
+
+func init() {
+
+	// var baseConfig string
+	var err error
+	var conf map[string]string
+	var MongoURL string
+	var DBname string
+
+	ResourceMap = make(map[string]interface{})
+	
+	logger := logs.CreateLogger("oclib","")
+
+	db_conf, err := os.ReadFile("tests/oclib_conf.json")
+	if err != nil {
+		logger.Fatal().Msg("Could not find configuration file")
+	}
+	json.Unmarshal(db_conf,&conf)
+	
+	if len(os.Getenv("DOCKER_ENVIRONMENT")) == 0 {
+		MongoURL = conf["DB_URL_LOCAL"]
+	} else {
+		MongoURL = conf["DB_URL_DOCKER"]
+	}
+
+	DBname = conf["DCNAME"] + "-" + conf["DBPOINT"]
+
+
+	logger.Info().Msg("Connecting to" + MongoURL)
+	
+	MngoCtx, cancel = context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	createClient(MongoURL)
+
+	logger.Info().Msg("Connecting mongo client to db " + DBname)
+	prepareDB(conf["DCNAME"],conf["DBPOINT"])
+
+	logger.Info().Msg("Database is READY")
+
+}
+
+func createClient(MongoURL string){
+
+	var err error
+
+	// Allows us to use marshal and unmarshall with results of FindOne() and others
+	bsonOpts := &options.BSONOptions {
+		UseJSONStructTags: true,
+		NilSliceAsEmpty: true,
+	}
+
+	clientOptions := options.Client().ApplyURI(MongoURL).SetBSONOptions(bsonOpts)
+	mngoClient, err = mongo.Connect(MngoCtx,clientOptions)
+	if err != nil {
+		logger.Fatal().Msg("Mongodb NewClient " + MongoURL + ":"  + "err" )
+		panic(err)
+	}
+
+
+	// Ping the primary
+	if mngoClient, err = mongo.Connect(MngoCtx, clientOptions); err != nil {
+		logger.Fatal().Msg("Mongodb connect " + MongoURL + ":"  + "err" )
+		panic(err)
+	}
+
+	if err = mngoClient.Ping(MngoCtx, nil); err != nil {
+		logger.Fatal().Msg("Mongodb ping " + MongoURL + ":"  + "err" )
+		panic(err)
+	}
+
+}
+
+func prepareDB(dc_name string, db_point string) {
+
+	var err error
+	DBname := dc_name + "-"  + db_point
+	mngoDB = mngoClient.Database(DBname)
+	
+	list_collection := [...]string{"data","processing","storage","datacenter","workspace","schedule","workflow"}
+
+	existingCollections, err = mngoDB.ListCollectionNames(MngoCtx,  bson.D{})
+	if err != nil {
+		logger.Fatal().Msg("Error contacting MongoDB\n" + err.Error())
+	}
+	collectionMap := make(map[string]bool)
+    for _, name := range existingCollections {
+        collectionMap[name] = true
+    }
+
+	// Only do the collection definition process if it doesn't already exists
+	// we add the collection to the collection map from mongo/mongo_utils to provide faster access to the collection
+
+	for _, collection_name := range(list_collection){
+		new_collection := mngoDB.Collection(collection_name)
+		if _, exists := collectionMap[collection_name]; !exists {
+			createCollection(collection_name, new_collection)
+		} else{
+			CollectionMap[collection_name] = new_collection
+		}
+	
+	}
+
+}
+
+// Creates the collection with index specified in mongo/mongo_collections
+// or use the basic collection creation function
+func createCollection(collection_name string, new_collection *mongo.Collection)  {
+
+	var err error 
+
+	CollectionMap[collection_name] = new_collection
+	
+	_, exists := IndexesMap[collection_name]; 
+	if exists{
+		if _, err = new_collection.Indexes().CreateMany(MngoCtx, IndexesMap[collection_name]); err != nil {
+			var cmdErr mongo.CommandError
+			if errors.As(err, &cmdErr) && cmdErr.Code != 85 {
+				logger.Fatal().Msg("Error creating indexes for " + collection_name + " collection : \n" + err.Error())
+				panic(err)
+			} else if !errors.As(err, &cmdErr) {
+				logger.Fatal().Msg("Unexpected error: " + err.Error())
+				panic(err)
+			}
+		}
+	} else {
+		mngoDB.CreateCollection(MngoCtx, collection_name)
+	}
+
+}
+
+func StoreOne(obj interface{}, collection_name string) (string, error) {
+	targetDBCollection := CollectionMap[collection_name]
+
+	MngoCtx, cancel = context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	result, err := targetDBCollection.InsertOne(MngoCtx,obj)
+	if err != nil {
+		logger.Error().Msg("Couldn't insert resource: " + err.Error())
+		return "", err
+	}
+
+	return result.InsertedID.(primitive.ObjectID).Hex(), nil
+}
+
+func LoadOne(id string, collection_name string) ( res *mongo.SingleResult , err error){
+	
+	// new_obj := ResourceMap[collection_name]
+	// var doc bson.D
+	
+	filter := bson.M{"_id": GetObjIDFromString(id)}
+	targetDBCollection := CollectionMap[collection_name]
+
+	MngoCtx, cancel = context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	res = targetDBCollection.FindOne(MngoCtx, filter)
+	
+	if res.Err() != nil {
+		logger.Error().Msg("Couldn't find resource " + id + ". Error : " + res.Err().Error())
+		err = res.Err()
+		return nil, err
+	}
+
+	return res, nil
+	
+}
+
+func GetObjIDFromString(id string) interface{} {
+	objectID, err := primitive.ObjectIDFromHex(id)
+	if err == nil {
+		return objectID
+	}
+
+	return id
+}
+

processing/processing.go

@@ -0,0 +1,32 @@
+package processing
+
+import oclib "oc-lib"
+
+type Processing struct {
+	oclib.AbstractResource    `json:"resource" required:"true"`
+	Container   string              `json:"container,omitempty"`  // We could create a specific class for container, that could check if the name exists/is available
+	Repository  string              `json:"repository,omitempty"` // Indicate where to find the container image	=> Could add a struct handling authentication to the repo
+	Command     string              `json:"command,omitempty"`
+	Arguments   []string            `json:"arguments,omitempty"`
+	Environment []map[string]string `json:"environment,omitempty"` // a key/value struct is what ressembles the most a NAME=VALUE struct
+
+	ExecutionRequirements ExecutionRequirementsModel `json:"execution_requirements,omitempty"`
+
+	Price   uint   `json:"price,omitempty"`
+	License string `json:"license,omitempty"`
+
+}
+
+type ExecutionRequirementsModel struct {
+	CPUs uint `json:"cp_us,omitempty"`
+	GPUs uint `json:"gp_us,omitempty"`
+	RAM  uint `json:"ram,omitempty"`
+
+	Parallel     bool   `json:"parallel,omitempty"`
+	ScalingModel uint   `json:"scaling_model,omitempty"`
+	DiskIO       string `json:"disk_io,omitempty"`
+}
+
+func (p *Processing) GetType() oclib.ResourceType{
+	return oclib.PROCESSING
+}

processing/processing_mongo_accessor.go

@@ -0,0 +1,38 @@
+package processing
+
+import (
+	"oc-lib/logs"
+	"oc-lib/mongo"
+)
+
+ type ProcessingMongoAccessor struct{
+
+ }
+
+
+func (pma *ProcessingMongoAccessor) StoreOne(processing Processing) string {
+	
+	id, err := mongo.StoreOne(processing,"processing")
+	if err != nil{
+		l := logs.CreateLogger("oclib","")
+		l.Error().Msg("Could not store " + processing.Name + " to db. Error: " + err.Error())
+		return ""
+	}
+	return id
+}
+
+func (pma *ProcessingMongoAccessor) LoadOne(id string) Processing {
+
+	var processing Processing
+
+	res_mongo, err := mongo.LoadOne(id,"processing")
+	if err != nil{
+			l := logs.CreateLogger("oclib","")
+			l.Error().Msg("Could not retrieve " + id + " from db. Error: " + err.Error())
+			return Processing{}
+		}
+
+	res_mongo.Decode(&processing)
+
+	return processing
+}

processing/processing_test.go

@@ -0,0 +1,48 @@
+package processing
+
+import (
+	oclib "oc-lib"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStoreOneProcessing(t *testing.T){
+	p := Processing{ Container: "totoCont", 
+				AbstractResource: oclib.AbstractResource{
+					Uuid: "123",
+					Name: "testData",
+					Description: "Lorem Ipsum",
+					Logo : "azerty.com",
+					Owner: "toto",
+					OwnerLogo: "totoLogo",
+					SourceUrl: "azerty.fr",
+				},
+			}
+
+	sma := ProcessingMongoAccessor{}
+	id := sma.StoreOne(p)
+
+	assert.NotEmpty(t, id)
+}
+
+func TestLoadOneProcessing(t *testing.T){
+	p := Processing{ Container: "totoCont", 
+				AbstractResource: oclib.AbstractResource{
+					Uuid: "123",
+					Name: "testData",
+					Description: "Lorem Ipsum",
+					Logo : "azerty.com",
+					Owner: "toto",
+					OwnerLogo: "totoLogo",
+					SourceUrl: "azerty.fr",
+				},
+			}
+
+	
+	sma := ProcessingMongoAccessor{}
+	id := sma.StoreOne(p)
+	new_s := sma.LoadOne(id)
+
+	assert.Equal(t,p, new_s)
+}

storage/storage.go

@@ -0,0 +1,24 @@
+package storage
+
+import oclib "oc-lib"
+
+type URL struct {
+	Protocol	string	`json:"protocol"`
+	Path		string	`json:"path"`
+}
+
+type Storage struct {
+	oclib.AbstractResource `json:"resource" required:"true" bson:"resource"`
+
+	Capacity uint `json:"capacity,omitempty"`
+	Url      URL  `json:"url,omitempty"` // Will allow to select between several protocols
+
+	Encryption   bool   `json:"encryption,omitempty"`
+	Redundancy   string `json:"redundancy,omitempty"`
+	Throughput   string `json:"throughput,omitempty"`
+	BookingPrice uint   `json:"booking_price,omitempty"`
+}
+
+func (s *Storage) GetType() oclib.ResourceType {
+	return oclib.STORAGE
+}

storage/storage_mongo_accessor.go

@@ -0,0 +1,38 @@
+package storage
+
+import (
+	"oc-lib/logs"
+	"oc-lib/mongo"
+)
+
+ type StorageMongoAccessor struct{
+
+ }
+
+
+func (schedulema *StorageMongoAccessor) StoreOne(storage Storage) string {
+	
+	id, err := mongo.StoreOne(storage,"storage")
+	if err != nil{
+		l := logs.CreateLogger("oclib","")
+		l.Error().Msg("Could not store " + storage.Name + " to db. Error: " + err.Error())
+		return ""
+	}
+	return id
+}
+
+func (schedulema *StorageMongoAccessor) LoadOne(id string) Storage {
+
+	var storage Storage
+
+	res_mongo, err := mongo.LoadOne(id,"storage")
+	if err != nil{
+			l := logs.CreateLogger("oclib","")
+			l.Error().Msg("Could not retrieve " + id + " from db. Error: " + err.Error())
+			return Storage{}
+		}
+
+	res_mongo.Decode(&storage)
+
+	return storage
+}

storage/storage_test.go

@@ -0,0 +1,48 @@
+package storage
+
+import (
+	oclib "oc-lib"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStoreOneStorage(t *testing.T){
+	s := Storage{ Capacity: 123, Url: URL{Protocol: "http",Path: "azerty.fr"} , 
+				AbstractResource: oclib.AbstractResource{
+					Uuid: "123",
+					Name: "testData",
+					Description: "Lorem Ipsum",
+					Logo : "azerty.com",
+					Owner: "toto",
+					OwnerLogo: "totoLogo",
+					SourceUrl: "azerty.fr",
+				},
+			}
+
+	sma := StorageMongoAccessor{}
+	id := sma.StoreOne(s)
+
+	assert.NotEmpty(t, id)
+}
+
+func TestLoadOneStorage(t *testing.T){
+	s := Storage{ Capacity: 123, Url: URL{Protocol: "http",Path: "azerty.fr"} , 
+				AbstractResource: oclib.AbstractResource{
+					Uuid: "123",
+					Name: "testData",
+					Description: "Lorem Ipsum",
+					Logo : "azerty.com",
+					Owner: "toto",
+					OwnerLogo: "totoLogo",
+					SourceUrl: "azerty.fr",
+				},
+			}
+
+	
+	sma := StorageMongoAccessor{}
+	id := sma.StoreOne(s)
+	new_s := sma.LoadOne(id)
+
+	assert.Equal(t,s, new_s)
+}

workflow/workflow.go

@@ -0,0 +1,34 @@
+package oclib
+
+import (
+	oclib "oc-lib"
+	"oc-lib/data"
+	"oc-lib/datacenter"
+	"oc-lib/link"
+	"oc-lib/processing"
+	"oc-lib/storage"
+)
+
+type Workflow struct {
+	oclib.AbstractResource `json:"abstract_resource" required:"true"`
+	Datas            map[string]data.Data       `json:"datas,omitempty"`
+	Storages         map[string]storage.Storage    `json:"storages,omitempty"`
+	Processing       map[string]processing.Processing `json:"processing,omitempty"`
+	Datacenters      map[string]datacenter.Datacenter `json:"datacenters,omitempty"`
+	Links            map[string]link.Link       `json:"links,omitempty"`
+
+	Schedule WorkflowSchedule `json:"schedule,omitempty"`
+}
+
+func (w *Workflow) isDCLink(link link.Link) bool {
+	if _, exists := w.Datacenters[link.Destination]; exists {
+        return true
+    } else if _, exists := w.Datacenters[link.Source]; exists {
+        return true
+    }
+	
+	return false
+}
+
+
+

workflow/workflow_mongo_accessor.go

@@ -0,0 +1,34 @@
+package oclib
+
+import (
+	"oc-lib/logs"
+	"oc-lib/mongo"
+)
+
+
+type WorkflowMongoAccessor struct{}
+
+func (wfa *WorkflowMongoAccessor) StoreOne(workflow Workflow) string {
+    id, err := mongo.StoreOne(workflow, "workflow")
+    if err != nil {
+        l := logs.CreateLogger("oclib", "")
+        l.Error().Msg("Could not store " + workflow.Name + " to db. Error: " + err.Error())
+        return ""
+    }
+    return id
+}
+
+func (wfa *WorkflowMongoAccessor) LoadOne(id string) Workflow {
+    var workflow Workflow
+
+    res_mongo, err := mongo.LoadOne(id, "workflow")
+    if err != nil {
+        l := logs.CreateLogger("oclib", "")
+        l.Error().Msg("Could not retrieve " + id + " from db. Error: " + err.Error())
+        return Workflow{}
+    }
+
+    res_mongo.Decode(&workflow)
+
+    return workflow
+}

workflow/workflow_schedule.go

@@ -0,0 +1,15 @@
+package oclib
+
+import "time"
+
+type WorkflowSchedule struct {
+	Id		string `json:"id"`
+	Start	time.Time 
+	End		time.Time
+	Cron	string
+}
+
+func (ws *WorkflowSchedule) GetAllDates() (timetable []time.Time){
+	// Return all the execution time generated by the Cron
+	return
+}

workflow/workflow_test.go

@@ -0,0 +1,45 @@
+package oclib
+
+import (
+	oclib "oc-lib"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStoreOneWorkflow(t *testing.T){
+	w := Workflow{AbstractResource: oclib.AbstractResource{
+					Uuid: "123",
+					Name: "testWorkflow",
+					Description: "Lorem Ipsum",
+					Logo : "azerty.com",
+					Owner: "toto",
+					OwnerLogo: "totoLogo",
+					SourceUrl: "azerty.fr",
+				},
+			}
+
+	wma := WorkflowMongoAccessor{}
+	id := wma.StoreOne(w)
+
+	assert.NotEmpty(t, id)
+}
+
+func TestLoadOneWorkflow(t *testing.T){
+	w := Workflow{AbstractResource: oclib.AbstractResource{
+					Uuid: "123",
+					Name: "testWorkflow",
+					Description: "Lorem Ipsum",
+					Logo : "azerty.com",
+					Owner: "toto",
+					OwnerLogo: "totoLogo",
+					SourceUrl: "azerty.fr",
+				},
+			}
+	
+	wma := WorkflowMongoAccessor{}
+	id := wma.StoreOne(w)
+	new_w := wma.LoadOne(id)
+
+	assert.Equal(t,w, new_w)
+}