First batch

This commit is contained in:
pb 2024-07-16 10:56:36 +02:00
parent 5ba33d3131
commit 034a81cedb
19 changed files with 540 additions and 30 deletions

1
.gitignore vendored Normal file
View File

@ -0,0 +1 @@
out/*

26
.vscode/launch.json vendored Normal file
View File

@ -0,0 +1,26 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Launch test function",
"type": "go",
"request": "launch",
"mode": "test",
"program": "${workspaceFolder}",
"args": [
"-test.run",
"TestMongoInit"
]
},
{
"name": "Launch Package",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${fileDirname}"
}
]
}

26
conf.go Normal file
View File

@ -0,0 +1,26 @@
package oclib
import "sync"
// ===================================================
// This class has to be updated everytime
// a new configuration variable is defined
// in a componant that imports oc-lib
// ===================================================
type Config struct {
MongoURL string
DCNAME string
DBPOINT string
}
var instance *Config
var once sync.Once
func GetConfig() *Config {
once.Do(func() {
instance = &Config{}
})
return instance
}

13
data.go
View File

@ -1,5 +1,16 @@
package oclib
type Data struct {
Resource
AbstractResource
Protocols []string `json:"protocol"` //TODO Enum type
DataType string `json:"datatype"`
Example string `json:"example" required:"true" validate:"required" description:"base64 encoded data"`
}
func (d *Data) GetType() ResourceType{
return DATA
}

9
data_test.go Normal file
View File

@ -0,0 +1,9 @@
package oclib
import (
"testing"
)
func TestPostOneData(t *testing.T){
}

View File

@ -1 +1,39 @@
package oclib
type Datacenter struct {
Resource
ID string `json:"ID", required: "true"`
Name string `json:"Name", required: "true" `
Owner string `json:"owner" `
BookingPrice int `json:"bookingPrice" `
CPU DatacenterCpuModel `json:"cpu" required:"true"`
RAM DatacenterMemoryModel `json:"ram" required:"true"`
GPU []DatacenterGpuModel `json:"gpu" required:"true"`
}
type DatacenterCpuModel struct {
Cores uint `json:"cores" required:"true"` //TODO: validate
Architecture string `json:"architecture"` //TOOD: enum
Shared bool `json:"shared"`
MinimumMemory uint `json:"minimum_memory"`
Platform string `json:"platform"`
}
type DatacenterMemoryModel struct {
Size uint `json:"size" description:"Units in MB"`
Ecc bool `json:"ecc"`
}
type DatacenterGpuModel struct {
CudaCores uint `json:"cuda_cores"`
Model string `json:"model"`
Memory uint `json:"memory" description:"Units in MB"`
TensorCores uint `json:"tensor_cores"`
}
func (d *Datacenter) GetType() ResourceType{
return DATACENTER
}

View File

@ -2,7 +2,7 @@
abstract Ressource {
+id: int
+UUID: int
+name: string
+icon: string
+description: string
@ -11,12 +11,12 @@ abstract Ressource {
}
class Data {
+id: int
+UUID: int
+name: string
}
class Processing {
+id: int
+UUID: int
+name: string
+container: string
+command: int
@ -25,83 +25,83 @@ class Processing {
}
class Storage {
+id: int
+UUID: int
+name: string
+url: string
+capacity: int
}
class Datacenter {
+id: int
+UUID: int
+name: string
}
class Workflow {
+id: int
+UUID: int
+name: string
}
class ResourceSet {
+id: int
+UUID: int
+name: string
+ressources: Ressource[]
}
class WorkflowSchedule {
+id: int
+UUID: int
+start: date
+end: date
+cron : string
}
class Graph {
+id: int
+ressources: map[GraphicElement.ID]Ressource
+UUID: int
+ressources: map[GraphicElement.UUID]Ressource
+links: Link[]
}
class Link {
+id: int
+source: GraphicElement.ID
+target: GraphicElement.ID
+UUID: int
+source: GraphicElement.UUID
+target: GraphicElement.UUID
+graphic: GraphicLink
}
class GraphicLink {
+id: int
+UUID: int
+startXY: coord
+endXY: coord
+style: string
}
class GraphicElement {
+id: int
+UUID: int
+style: string
+xy: coord
}
class Calendar {
+id: int
+UUID: int
+name: string
+workflows: Workflow[]
+owner: string
}
class UserWorkflows {
+id: int
+UUID: int
+user: string
+workflows: Workflow[]
}
class DatacenterWorkflows {
+id: int
+UUID: int
+datacenter: Datacenter
+workflows: Workflow[]
}
class Graph {
+id: int
+UUID: int
+graph: Graph
+workflows: Workflow[]
}
@ -121,7 +121,7 @@ Ressource <|-- Workflow
ResourceSet "1" o-- "0..*" Ressource
Workflow "1" o-- "0..*" ResourceSet
Workflow "1" o-- "0..*" WorkflowSchedule
Workflow "1" o-- "0..1" WorkflowSchedule
Workflow "1" o-- "0..*" Graph
Graph "1" o-- "0..*" Resources

Binary file not shown.

Before

Width:  |  Height:  |  Size: 89 KiB

After

Width:  |  Height:  |  Size: 102 KiB

12
graphic_element.go Normal file
View File

@ -0,0 +1,12 @@
package oclib
type Coordinate struct {
x int
y int
}
type GraphicElement struct{
ID string `json:"ID" required:"true"`
style string `json:"style" required:"true"`
xy Coordinate `json:"xy" required:"true"`
}

7
link.go Normal file
View File

@ -0,0 +1,7 @@
package oclib
type Link struct {
Source string
Destination string
}

187
mongo.go Normal file
View File

@ -0,0 +1,187 @@
package oclib
import (
"context"
"encoding/json"
"errors"
"os"
"time"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/x/bsonx"
)
var (
mngoClient *mongo.Client
mngoDB *mongo.Database
MngoCtx context.Context
MngoCollData *mongo.Collection
MngoCollComputing *mongo.Collection
MngoCollStorage *mongo.Collection
MngoCollDatacenter *mongo.Collection
MngoCollWorkspace *mongo.Collection
MngoCollSchedule *mongo.Collection
)
// Trying to get vscode to display this
func MongoInit() {
// var baseConfig string
var err error
var cancel context.CancelFunc
var conf map[string]string
var MongoURL string
var DBname string
db_conf, err := os.ReadFile("oclib_conf.json")
if err != nil {
logger.Fatal().Msg("Could not find configuration file")
}
json.Unmarshal(db_conf,&conf)
if len(os.Getenv("DOCKER_ENVIRONMENT")) == 0 {
MongoURL = conf["DB_URL_LOCAL"]
} else {
MongoURL = conf["DB_URL_DOCKER"]
}
DBname = conf["DCNAME"] + "-" + conf["DBPOINT"]
// MongoURL, err := beego.AppConfig.String(baseConfig + "::url")
// if err != nil {
// logger.Critical().Msg("MongoDB URI error: %v", err)
// panic(err)
// }
logger.Info().Msg("Connecting to" + MongoURL)
MngoCtx, cancel = context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
createClient(MongoURL)
logger.Info().Msg("Connecting mongo client to db " + DBname)
prepareDB(conf["DCNAME"],conf["DBPOINT"])
logger.Info().Msg("Database is READY")
}
// TODO : Soit retourner MngoCtx, soit comprendre comment passer sa référence et la mettre à jour
func createClient(MongoURL string){
var err error
clientOptions := options.Client().ApplyURI(MongoURL)
mngoClient, _ = mongo.NewClient(options.Client().ApplyURI(MongoURL))
if err = mngoClient.Connect(MngoCtx); err != nil {
logger.Fatal().Msg("Mongodb NewClient " + MongoURL + ":" + "err" )
panic(err)
}
// Ping the primary
if mngoClient, err = mongo.Connect(MngoCtx, clientOptions); err != nil {
logger.Fatal().Msg("Mongodb connect " + MongoURL + ":" + "err" )
panic(err)
}
if err = mngoClient.Ping(MngoCtx, nil); err != nil {
logger.Fatal().Msg("Mongodb ping " + MongoURL + ":" + "err" )
panic(err)
}
}
func prepareDB(dc_name string, db_point string) {
// var err error
DBname := dc_name + "-" + db_point
mngoDB = mngoClient.Database(DBname)
MngoCollData = mngoDB.Collection("Data")
MngoCollComputing = mngoDB.Collection("Computing")
MngoCollStorage = mngoDB.Collection("Storage")
MngoCollDatacenter = mngoDB.Collection("Datacenter")
MngoCollWorkspace = mngoDB.Collection("Workspace")
MngoCollSchedule = mngoDB.Collection("Schedule")
list_collection := [...]string{"Data","Computing","Storage","Datacenter","Workspace","Schedule"}
for _, collection_name := range(list_collection){
new_collection := mngoDB.Collection(collection_name)
createCollection(new_collection)
}
// if _, err = MngoCollComputing.Indexes().CreateMany(MngoCtx, []mongo.IndexModel{
// {
// Keys: bsonx.Doc{
// {Key: "description", Value: bsonx.String("text")},
// {Key: "owner", Value: bsonx.String("text")},
// {Key: "license", Value: bsonx.String("text")},
// },
// },
// }); err != nil && err.(mongo.CommandError).Code != 85 {
// logger.Critical().Msg(err)
// panic(err)
// }
// if _, err = MngoCollStorage.Indexes().CreateMany(MngoCtx, []mongo.IndexModel{
// {
// Keys: bsonx.Doc{
// {Key: "name", Value: bsonx.String("text")},
// {Key: "description", Value: bsonx.String("text")},
// },
// },
// }); err != nil && err.(mongo.CommandError).Code != 85 {
// logger.Critical().Msg(err)
// panic(err)
// }
// if _, err = MngoCollDatacenter.Indexes().CreateMany(MngoCtx, []mongo.IndexModel{
// {
// Keys: bsonx.Doc{
// {Key: "name", Value: bsonx.String("text")},
// {Key: "description", Value: bsonx.String("text")},
// {Key: "owner", Value: bsonx.String("text")},
// },
// },
// }); err != nil && err.(mongo.CommandError).Code != 85 {
// logger.Critical().Msg(err)
// panic(err)
// }
}
func createCollection(new_collection *mongo.Collection) {
var err error
if _, err = new_collection.Indexes().CreateMany(MngoCtx, []mongo.IndexModel{
{
Keys: bsonx.Doc{
{Key: "description", Value: bsonx.String("text")},
{Key: "example", Value: bsonx.String("text")},
},
},
}); err != nil {
var cmdErr mongo.CommandError
if errors.As(err, &cmdErr) && cmdErr.Code != 85 {
logger.Fatal().Msg("It failed but I saw it: " + err.Error())
panic(err)
} else if !errors.As(err, &cmdErr) {
logger.Fatal().Msg("Unexpected error: " + err.Error())
panic(err)
}
}
}

10
mongo_test.go Normal file
View File

@ -0,0 +1,10 @@
package oclib
import (
"fmt"
"testing"
)
func TestMongoInit(T *testing.T){
MongoInit()
fmt.Printf("It worked !")
}

6
oclib_conf.json Normal file
View File

@ -0,0 +1,6 @@
{
"DB_URL_LOCAL" : "mongodb://127.0.0.1:27017",
"DB_URL_DOCKER": "mongodb://mongo:27017/",
"DBPOINT" : "oclib_tests",
"DCNAME" : "testDC"
}

30
processing.go Normal file
View File

@ -0,0 +1,30 @@
package oclib
type Processing struct {
Resource
Container string `json:"Container"` // We could create a specific class for container, that could check if the name exists/is available
Repository string `json:"Repository"` // Indicate where to find the container image => Could add a struct handling authentication to the repo
Command string `json:"Command"`
Arguments []string `json:"Arguments"`
Environment []map[string]string `json:"Environment"` // a key/value struct is what ressembles the most a NAME=VALUE struct
ExecutionRequirements ExecutionRequirementsModel `json:"ExecutionRequirements"`
Price uint `json:"Price"`
License string `json:"License"`
}
type ExecutionRequirementsModel struct {
CPUs uint `json:"cpus" required:"true"`
GPUs uint `json:"gpus" description:"Amount of GPUs needed"`
RAM uint `json:"ram" required:"true" description:"Units in MB"`
Parallel bool `json:"parallel"`
ScalingModel uint `json:"scaling_model"`
DiskIO string `json:"disk_io"`
}
func (p *Processing) GetType() ResourceType{
return PROCESSING
}

View File

@ -1,12 +1,94 @@
package oclib
type Resource struct {
Id string
Name string
import (
"go.mongodb.org/mongo-driver/bson/primitive"
)
// AbstractResource is the struct containing all of the attributes commons to all ressources
// Resource is the interface to be implemented by all classes inheriting from Resource to have the same behavior
//http://www.inanzzz.com/index.php/post/wqbs/a-basic-usage-of-int-and-string-enum-types-in-golang
type ResourceType int
const (
INVALID ResourceType = iota
DATA
PROCESSING
STORAGE
DATACENTER
WORKFLOW
)
var extensions = [...]string{
"INVALID",
"DATA",
"PROCESSING",
"STORAGE",
"DATACENTER",
"WORKFLOW",
}
type Resource interface{
GetType() ResourceType
getOneResourceByID() Resource
}
type AbstractResource struct {
Uuid string `json:"Id" required:"true" `
Name string `json:"Name" required:"true" `
ShortDescription string
Description string
Logo string
Owner string
OwnerLogo string
SourceUrl string
Graphic GraphicElement `json:"GraphicElement" `
}
// func (r *AbstractResource) getOneResourceByID(id string, resType ResourceType){
// targetDBCollection := r.GetType().MongoCollection() // Change the rType by the result of reflect
// var retObj interface{}
// filter := bson.M{"_id": getObjIDFromString(id)}
// res := targetDBCollection.FindOne(services.MngoCtx, filter)
// res.Decode(retObj)
// if res.Err() != nil {
// logs.Warn("Couldn't find resource: " + res.Err().Error())
// }
// return retObj, res.Err()
// }
func getObjIDFromString(id string) interface{} {
objectID, err := primitive.ObjectIDFromHex(id)
if err == nil {
return objectID
}
return id
}
func postToMongo(id string) {
}
func (r *AbstractResource) isLinked(){
// Get the link collection in this workflow
// test if the current resource is a dest OR a source at least one
// (len(slice[r.ID == dest]) > 0 || len(slice[r.ID == 1]) > 1 )
}
// func (r *Resource) GetType() ResourceType {
// return INVALID
// }

View File

@ -1 +1,6 @@
package oclib
// Resources' key must be the Resource' Uuid, to garanty the set-like structure
type WorkflowSet struct{
Resources map[string]Resource
}

View File

@ -1 +1,22 @@
package oclib
type URL struct {
Protocol string `json:"protocol"`
Path string `json:"path"`
}
type Storage struct {
Resource `json:"resource" required:"true"`
Capacity uint `json:"capacity" required:"true"`
Url URL `json:"URL" ` // Will allow to select between several protocols
Encryption bool `json:"encryption" `
Redundancy string `json:"redundancy" `
Throughput string `json:"throughput" `
BookingPrice uint `json:"bookingPrice" `
}
func (s *Storage) GetType() ResourceType{
return STORAGE
}

25
workflow.go Normal file
View File

@ -0,0 +1,25 @@
package oclib
type Workflow struct{
Resource
Datas map[string]Data
Storages map[string]Storage
Processing map[string]Processing
Datacenters map[string]Datacenter
Links map[string]Link
Schedule WorkflowSchedule
}
func (w *Workflow) isDCLink(link Link) bool {
if _, exists := w.Datacenters[link.Destination]; exists {
return true
} else if _, exists := w.Datacenters[link.Source]; exists {
return true
}
return false
}

View File

@ -1 +1,15 @@
package oclib
import "time"
type WorkflowSchedule struct {
Id string `json:"id"`
Start time.Time
End time.Time
Cron string
}
func (ws *WorkflowSchedule) GetAllDates() (timetable []time.Time){
// Return all the execution time generated by the Cron
return
}