From c6d20aec3325b70752e1e76f0672e0e639505a5e Mon Sep 17 00:00:00 2001 From: pb Date: Thu, 29 Aug 2024 10:17:31 +0200 Subject: [PATCH] adapt to services --- demo_nginx/workflow_nginx_services.json | 292 ++++++++++++++++++++++++ go.mod | 2 +- go.sum | 3 + main.go | 14 +- workflow_builder/argo_builder.go | 50 +++- 5 files changed, 353 insertions(+), 8 deletions(-) create mode 100644 demo_nginx/workflow_nginx_services.json diff --git a/demo_nginx/workflow_nginx_services.json b/demo_nginx/workflow_nginx_services.json new file mode 100644 index 0000000..acbee01 --- /dev/null +++ b/demo_nginx/workflow_nginx_services.json @@ -0,0 +1,292 @@ +{ + "id": "9c7ffc7e-3e6e-4ea8-8eab-3a03258712ff", + "name": "test-services", + "resourceset": { + "processings": [ + "7c71a15b-bdbc-46d7-9dab-67e369804136", + "0d565c87-50ae-4a73-843d-f8b2d4047772" + ], + "datacenters": [ + "7b989e97-c3e7-49d2-a3a7-f959da4870b5" + ] + }, + "graph": { + "zoom": 1, + "items": { + "aa8d2265-9fe2-42c7-ba1f-46ea0da8e633": { + "id": "aa8d2265-9fe2-42c7-ba1f-46ea0da8e633", + "width": 0, + "height": 0, + "processing": { + "id": "0d565c87-50ae-4a73-843d-f8b2d4047772", + "resource_model": { + "resource_type": "processing", + "model": { + "command": { + "type": "string", + "value": "curlimages/curl:7.88.1" + }, + "args": { + "type": "string", + "value": "-SL https://cloud.o-forge.io/core/oc-monitord/raw/branch/services_demo/demo_nginx/cockpit.html -o /usr/share/ningx/cockpit.hmtl" + } + } + }, + "name": "CURL", + "short_description": "Transfer or retrieve information from or to a server ", + "description": "curl is a tool for transferring data from or to a server. It supports these protocols: DICT, FILE, FTP, FTPS, GOPHER, GOPHERS, HTTP, HTTPS, IMAP, IMAPS, LDAP, LDAPS, MQTT, POP3, POP3S, RTMP, RTMPS, RTSP, SCP, SFTP, SMB, SMBS, SMTP, SMTPS, TELNET, TFTP, WS and WSS.", + "logo": "https://cloud.o-forge.io/core/deperecated-oc-catalog/src/branch/main/scripts/local_imgs/curl-logo.png", + "owner": "IRT", + "source_url": "http://www.google.com", + "license": "GPLv2", + "ram": {}, + "storage": 300, + "parallel": true, + "scaling_model": 2, + "disk_io": "30 MB/s" + } + }, + "a2d273c1-e564-45ad-a720-c9a40c28c6b5": { + "id": "a2d273c1-e564-45ad-a720-c9a40c28c6b5", + "width": 0, + "height": 0, + "processing": { + "id": "0d565c87-50ae-4a73-843d-f8b2d4047772", + "resource_model": { + "resource_type": "processing", + "model": { + "command": { + "type": "string", + "value": "curlimages/curl:7.88.1" + }, + "args": { + "type": "string", + "value": "-SL https://cloud.o-forge.io/core/oc-monitord/raw/branch/services_demo/demo_nginx/DTF.html -o /usr/share/ningx/DTF.hmtl" + } + } + }, + "name": "CURL", + "short_description": "Transfer or retrieve information from or to a server ", + "description": "curl is a tool for transferring data from or to a server. It supports these protocols: DICT, FILE, FTP, FTPS, GOPHER, GOPHERS, HTTP, HTTPS, IMAP, IMAPS, LDAP, LDAPS, MQTT, POP3, POP3S, RTMP, RTMPS, RTSP, SCP, SFTP, SMB, SMBS, SMTP, SMTPS, TELNET, TFTP, WS and WSS.", + "logo": "https://cloud.o-forge.io/core/deperecated-oc-catalog/src/branch/main/scripts/local_imgs/curl-logo.png", + "owner": "IRT", + "source_url": "http://www.google.com", + "license": "GPLv2", + "ram": {}, + "storage": 300, + "parallel": true, + "scaling_model": 2, + "disk_io": "30 MB/s" + } + }, + "bf6916ff-b16f-44b3-818b-0bcd5bbaca00": { + "id": "bf6916ff-b16f-44b3-818b-0bcd5bbaca00", + "width": 0, + "height": 0, + "position": { + "id": "", + "x": 0, + "y": 0 + }, + "processing": { + + "id": "5234c921-490f-40b1-ab1e-56570dd7879e", + "resource_model": { + "resource_type": "processing", + "model": { + "command": { + "type": "string", + "value": "nginx" + }, + "args": { + "type": "string" + } + } + }, + "name": "NGINX", + "short_description": "an open source reverse proxy server for HTTP, HTTPS, SMTP, POP3, and IMAP protocols, as well as a load balancer, HTTP cache, and a web server ( ", + "description": "Nginx (pronounced 'engine-x') is an open source reverse proxy server for HTTP, HTTPS, SMTP, POP3, and IMAP protocols, as well as a load balancer, HTTP cache, and a web server (origin server). The nginx project started with a strong focus on high concurrency, high performance and low memory usage. It is licensed under the 2-clause BSD-like license and it runs on Linux, BSD variants, Mac OS X, Solaris, AIX, HP-UX, as well as on other *nix flavors. It also has a proof of concept port for Microsoft Windows.", + "logo": "https://cloud.o-forge.io/core/deperecated-oc-catalog/raw/branch/main/scripts/local_imgs/nginx.png", + "owner": "IRT", + "source_url": "https://hub.docker.com/_/nginx", + "license": "GPLv2", + "ram": {}, + "storage": 300, + "parallel": true, + "scaling_model": 2, + "disk_io": "30 MB/s" + + } + }, + "d83ac451-4690-44d9-af09-48e7588b2db9": { + "id": "d83ac451-4690-44d9-af09-48e7588b2db9", + "width": 0, + "height": 0, + "position": { + "id": "", + "x": 0, + "y": 0 + }, + "processing": { + + "id": "5234c921-490f-40b1-ab1e-56570dd7879e", + "resource_model": { + "resource_type": "processing", + "model": { + "command": { + "type": "string", + "value": "nginx" + }, + "args": { + "type": "string" + } + } + }, + "name": "NGINX", + "short_description": "an open source reverse proxy server for HTTP, HTTPS, SMTP, POP3, and IMAP protocols, as well as a load balancer, HTTP cache, and a web server ( ", + "description": "Nginx (pronounced 'engine-x') is an open source reverse proxy server for HTTP, HTTPS, SMTP, POP3, and IMAP protocols, as well as a load balancer, HTTP cache, and a web server (origin server). The nginx project started with a strong focus on high concurrency, high performance and low memory usage. It is licensed under the 2-clause BSD-like license and it runs on Linux, BSD variants, Mac OS X, Solaris, AIX, HP-UX, as well as on other *nix flavors. It also has a proof of concept port for Microsoft Windows.", + "logo": "https://cloud.o-forge.io/core/deperecated-oc-catalog/raw/branch/main/scripts/local_imgs/nginx.png", + "owner": "IRT", + "source_url": "https://hub.docker.com/_/nginx", + "license": "GPLv2", + "ram": {}, + "storage": 300, + "parallel": true, + "scaling_model": 2, + "disk_io": "30 MB/s" + + } + }, + "6a7e8860-7c26-4b70-9b3a-1bd27adcdfe1": { + "id": "6a7e8860-7c26-4b70-9b3a-1bd27adcdfe1", + "width": 0, + "height": 0, + "position": { + "id": "", + "x": 0, + "y": 0 + }, + "itemresource": { + "datacenter": { + "cpus": [ + { + "architecture": "x86", + "cores": { + "$numberLong": "8" + } + } + ], + "ram": { + "size": { + "$numberLong": "16384" + } + }, + "gpus": [ + { + "memory": { + "$numberLong": "24000" + }, + "tensor_cores": { + "$numberLong": "328" + }, + "cuda_cores": { + "$numberLong": "10496" + }, + "model": "RTX 3090 FE" + } + ], + "abstractresource": { + "abstractobject": { + "id": "7b989e97-c3e7-49d2-a3a7-f959da4870b5", + "name": "Mundi datacenter" + }, + "short_description": "Mundi Opencloud Instance", + "description": "A very long description of what this data is", + "logo": "https://cloud.o-forge.io/core/deperecated-oc-catalog/src/branch/main/scripts/local_imgs/Mundi datacenter.png", + "owner": "IRT", + "source_url": "http://www.google.com", + "resource_model": { + "id": "c3983010-1990-4ac0-8533-5389867e4424", + "resource_type": "datacenter_resource" + } + } + } + } + } + }, + "links": [ + { + "source": { + "id": "aa8d2265-9fe2-42c7-ba1f-46ea0da8e633", + "x": 0, + "y": 0 + }, + "destination": { + "id": "0d565c87-50ae-4a73-843d-f8b2d4047772", + "x": 0, + "y": 0 + } + }, + { + "source": { + "id": "a2d273c1-e564-45ad-a720-c9a40c28c6b5", + "x": 0, + "y": 0 + }, + "destination": { + "id": "d83ac451-4690-44d9-af09-48e7588b2db9", + "x": 0, + "y": 0 + } + }, + { + "source": { + "id": "a2d273c1-e564-45ad-a720-c9a40c28c6b5", + "x": 0, + "y": 0 + }, + "destination": { + "id": "6a7e8860-7c26-4b70-9b3a-1bd27adcdfe1", + "x": 0, + "y": 0 + } + }, + { + "source": { + "id": "aa8d2265-9fe2-42c7-ba1f-46ea0da8e633", + "x": 0, + "y": 0 + }, + "destination": { + "id": "6a7e8860-7c26-4b70-9b3a-1bd27adcdfe1", + "x": 0, + "y": 0 + } + }, + { + "source": { + "id": "bf6916ff-b16f-44b3-818b-0bcd5bbaca00", + "x": 0, + "y": 0 + }, + "destination": { + "id": "6a7e8860-7c26-4b70-9b3a-1bd27adcdfe1", + "x": 0, + "y": 0 + } + }, + { + "source": { + "id": "6a7e8860-7c26-4b70-9b3a-1bd27adcdfe1", + "x": 0, + "y": 0 + }, + "destination": { + "id": "d83ac451-4690-44d9-af09-48e7588b2db9", + "x": 0, + "y": 0 + } + } + ] + } +} \ No newline at end of file diff --git a/go.mod b/go.mod index 1506665..8d50990 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module oc-monitord go 1.22.0 require ( - cloud.o-forge.io/core/oc-lib v0.0.0-20240826085916-d0e1474f8f34 + cloud.o-forge.io/core/oc-lib v0.0.0-20240828135227-14d6a5f11c4e github.com/akamensky/argparse v1.4.0 github.com/goraz/onion v0.1.3 github.com/nats-io/nats-server/v2 v2.10.18 diff --git a/go.sum b/go.sum index 7eff7ab..6bf2dc7 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,8 @@ cloud.o-forge.io/core/oc-lib v0.0.0-20240822081914-4abf59a10d97 h1:6tbeTQvRnD0vD cloud.o-forge.io/core/oc-lib v0.0.0-20240822081914-4abf59a10d97/go.mod h1:1hhYh5QWAbYw9cKplQ0ZD9PMgU8t6gPqiYF8sldv1HU= cloud.o-forge.io/core/oc-lib v0.0.0-20240826085916-d0e1474f8f34 h1:40XQgwR9HxXSnouY+ZqE/xYCM4qa+U+RLA5GA5JSNyQ= cloud.o-forge.io/core/oc-lib v0.0.0-20240826085916-d0e1474f8f34/go.mod h1:1hhYh5QWAbYw9cKplQ0ZD9PMgU8t6gPqiYF8sldv1HU= +cloud.o-forge.io/core/oc-lib v0.0.0-20240828135227-14d6a5f11c4e h1:/KWO/gIcP5f7T4r00715fNz0Y/Hil6Bj3J1ycuES1Zw= +cloud.o-forge.io/core/oc-lib v0.0.0-20240828135227-14d6a5f11c4e/go.mod h1:FIJD0taWLJ5pjQLJ6sfE2KlTkvbmk5SMcyrxdjsaVz0= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/akamensky/argparse v1.4.0 h1:YGzvsTqCvbEZhL8zZu2AiA5nq805NZh75JNj4ajn1xc= github.com/akamensky/argparse v1.4.0/go.mod h1:S5kwC7IuDcEr5VeXtGPRVZ5o/FdhcMlQz4IZQuw64xA= @@ -221,6 +223,7 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go index 7d0da97..580c04e 100644 --- a/main.go +++ b/main.go @@ -28,6 +28,13 @@ import ( "github.com/rs/zerolog" ) +// Command-line args: +// - url: Loki URL (default: "http://127.0.0.1:3100") +// - execution: Workflow Execution ID (required) to identify the current execution, allows to retrieve Workflow +// - mongo: MongoDB URL (default: "mongodb://127.0.0.1:27017") +// - db: MongoDB database name (default: "DC_myDC") +// - timeout: Execution timeout (default: -1) + var logger zerolog.Logger var wf_logger zerolog.Logger var pods_logger zerolog.Logger @@ -40,6 +47,8 @@ const localConfigFile = "./conf/local_ocmonitord_conf.json" func main() { + os.Setenv("test_service","true") // Only for service demo, delete before merging on main + monitorLocal = false // Test if monitor is launched outside (with parameters) or in a k8s environment (env variables sets) if os.Getenv("KUBERNETES_SERVICE_HOST") == "" { @@ -216,7 +225,7 @@ func setConf(is_k8s bool, o *onion.Onion, parser *argparse.Parser) { tools.SetConfig(mongo, db, "") } else { url := parser.String("u", "url", &argparse.Options{Required: true, Default: "http://127.0.0.1:3100", Help: "Url to the Loki database logs will be sent to"}) - workflow := parser.String("w", "workflow", &argparse.Options{Required: true, Help: "Execution ID of the workflow to request from oc-catalog API"}) + execution := parser.String("e", "execution", &argparse.Options{Required: true, Help: "Execution ID of the workflow to request from oc-catalog API"}) mongo := parser.String("m", "mongo", &argparse.Options{Required: true, Default: "mongodb://127.0.0.1:27017", Help: "URL to reach the MongoDB"}) db := parser.String("d", "database", &argparse.Options{Required: true, Default: "DC_myDC", Help: "Name of the database to query in MongoDB"}) timeout := parser.Int("t", "timeout", &argparse.Options{Required: false, Default: -1, Help: "Timeout for the execution of the workflow"}) @@ -227,12 +236,13 @@ func setConf(is_k8s bool, o *onion.Onion, parser *argparse.Parser) { } conf.GetConfig().LokiURL = *url conf.GetConfig().Timeout = *timeout - conf.GetConfig().ExecutionID = *workflow + conf.GetConfig().ExecutionID = *execution tools.SetConfig(*mongo, *db, "") } } + func initOnion(o *onion.Onion) *onion.Onion { logger = logs.CreateLogger("oc-monitord", "") configFile := "" diff --git a/workflow_builder/argo_builder.go b/workflow_builder/argo_builder.go index 8abf13e..fffbc50 100644 --- a/workflow_builder/argo_builder.go +++ b/workflow_builder/argo_builder.go @@ -41,9 +41,17 @@ type Spec struct { } func (b *ArgoBuilder) CreateDAG() (string, error) { + + // handle services by checking if there is only one processing with hostname and port + if (b.isService()){ + b.createNginxVolumes() + } + b.createTemplates() b.createDAGstep() b.createVolumes() + + if b.Timeout > 0 { b.Workflow.Spec.Timeout = b.Timeout } @@ -87,7 +95,7 @@ func (b *ArgoBuilder) createTemplates() { image_name := strings.Split(command, " ")[0] // TODO : decide where to store the image name, GUI or models.computing.Image temp_container := Container{Image: image_name} // TODO : decide where to store the image name, GUI or models.computing.Image temp_container.Command = getComputingCommands(command) - temp_container.Args = getComputingArgs(strings.Split(args, " "), command) + temp_container.Args = getComputingArgs(args, command) // Only for dev purpose, input_names := getComputingEnvironmentName(strings.Split(env, " ")) @@ -100,6 +108,7 @@ func (b *ArgoBuilder) createTemplates() { new_temp := Template{Name: argo_name, Container: temp_container} new_temp.Inputs.Parameters = inputs_container new_temp.Container.VolumeMounts = append(new_temp.Container.VolumeMounts, VolumeMount{Name: "workdir", MountPath: "/mnt/vol"}) // TODO : replace this with a search of the storage / data source name + new_temp.Container.VolumeMounts = append(new_temp.Container.VolumeMounts, VolumeMount{Name: "nginx-demo", MountPath: "/usr/share/nginx"}) // Used for processing services' demo with nginx b.Workflow.Spec.Templates = append(b.Workflow.Spec.Templates, new_temp) } @@ -136,9 +145,20 @@ func (b *ArgoBuilder) createVolumes() { b.Workflow.Spec.Volumes = append(b.Workflow.Spec.Volumes, new_volume) } +// For demo purposes, until we implement the use of storage ressources +func (b *ArgoBuilder) createNginxVolumes() { + new_volume := VolumeClaimTemplate{} + new_volume.Metadata.Name = "nginx-demo" + new_volume.Spec.AccessModes = []string{"ReadWriteOnce"} + new_volume.Spec.Resources.Requests.Storage = "1Gi" + + b.Workflow.Spec.Volumes = append(b.Workflow.Spec.Volumes, new_volume) +} + + func (b *ArgoBuilder) getDependency(current_computing_id string) (dependencies []string) { for _, link := range b.graph.Links { - if !b.IsProcessing(link.Source.ID) || !b.IsProcessing(link.Destination.ID) { + if !b.IsProcessing(link.Source.ID) { continue } source := b.graph.Items[link.Source.ID].Processing @@ -159,17 +179,20 @@ func getComputingCommands(user_input string) []string { return strings.Split(user_input, " ") } -func getComputingArgs(user_input []string, command string) (list_args []string) { +func getComputingArgs(user_input string, command string) (list_args []string) { if len(user_input) == 0 { return } + + args := strings.Split(user_input," ") + // quickfix that might need improvement if strings.Contains(command, "sh -c") { - list_args = append(list_args, strings.Join(user_input, " ")) + list_args = append(list_args, strings.Join(args, " ")) return } - list_args = append(list_args, user_input...) + list_args = append(list_args, args...) return } @@ -262,3 +285,20 @@ func getStringValue(comp resource_model.AbstractResource, key string) string { } return "" } + +func (b *ArgoBuilder) isService() bool{ + // for dev purpose do not commit to main + if os.Getenv("test_service") != ""{ + return true + } + + comp_list := b.getProcessings() + + if len(comp_list) != 1 { + return false + } + + comp := comp_list[0] + + return comp.Data.ResourceModel.Model["port"].Value != "" && comp.Data.ResourceModel.Model["hostname"].Value != "" +} \ No newline at end of file