From cd6d5a24aa7b0677997e61060c274fca474239c5 Mon Sep 17 00:00:00 2001 From: pb Date: Fri, 6 Jun 2025 16:02:23 +0200 Subject: [PATCH 1/9] added documentation for this feature implementation --- minio-doc/README.md | 66 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 minio-doc/README.md diff --git a/minio-doc/README.md b/minio-doc/README.md new file mode 100644 index 0000000..51d1b67 --- /dev/null +++ b/minio-doc/README.md @@ -0,0 +1,66 @@ +# Goal + +We want to be able to instantiate a service that allows to store file located on a `processing` pod onto it. + +We have already tested it with a static `Argo` yaml file, a MinIO running on the same kubernetes node, the minio service is reached because it is the only associated to the `serviceAccount`. + +We have established three otpions that need to be available to the user for the feature to be implemented: + +- Use a MinIO running constantly on the node that executes the argo workflow +- Use a MinIO +- A MinIO is instanciated when a new workflow is launched + +# Requirements + +- Helm : `https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3` +- Helm GO client : `$ go get github.com/mittwald/go-helm-client` +- MinIO chart : `https://charts.min.io/` + + +# Ressources + +We need to create several ressources in order for the pods to communicate with the MinIO + +## MinIO Auth Secrets + +## Bucket ConfigMap + +With the name `artifact-repositories` this configMap will be used by default. It contains the URL to the MinIO server and the key to the authentication data held in a `secret` ressource. + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + # If you want to use this config map by default, name it "artifact-repositories". + name: artifact-repositories + # annotations: + # # v3.0 and after - if you want to use a specific key, put that key into this annotation. + # workflows.argoproj.io/default-artifact-repository: oc-s3-artifact-repository +data: + oc-s3-artifact-repository: | + s3: + bucket: oc-bucket + endpoint: [ retrieve cluster with kubectl get service argo-artifacts -o jsonpath="{.spec.clusterIP}" ]:9000 + insecure: true + accessKeySecret: + name: argo-artifact-secret + key: access-key + secretKeySecret: + name: argo-artifact-secret + key: secret-key + +``` + + +# Code modifications + +Rajouter un attribut "isDataLink" + - true/false + +Rajouter un attribut DataPath ou un truc comme ca + + - liste de map[string]string permet de n'avoir qu'une copie par fichier) + - éditable uniquement a travers la méthode addDataPath + - clé : path du fichier / value : nom de la copie dans minio + +===> on a besoin du meme attribut pour Processing -> Data et Data -> Processing \ No newline at end of file From f5e1707545a12cd1270ebdbb0fd28b794cef74c1 Mon Sep 17 00:00:00 2001 From: pb Date: Mon, 16 Jun 2025 18:21:34 +0200 Subject: [PATCH 2/9] corrected the yaml output of accesskey and secret key + added another condition to ReplacePerEnv --- models/template.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/template.go b/models/template.go index f7a6019..05dde12 100644 --- a/models/template.go +++ b/models/template.go @@ -71,8 +71,8 @@ type Key struct { Bucket string `yaml:"bucket"` EndPoint string `yaml:"endpoint"` Insecure bool `yaml:"insecure"` - AccessKeySecret *Secret `yaml accessKeySecret` - SecretKeySecret *Secret `yaml secretKeySecret` + AccessKeySecret *Secret `yaml: accessKeySecret` + SecretKeySecret *Secret `yaml: secretKeySecret` } type Artifact struct { @@ -131,7 +131,7 @@ func (template *Template) CreateContainer(processing *resources.ProcessingResour func (template *Template) ReplacePerEnv(arg string, envs []models.Param) string { for _, v := range envs { - if strings.Contains(arg, v.Name) { + if v.Name != "" && strings.Contains(arg, v.Name) { value := "{{ inputs.parameters." + v.Name + " }}" arg = strings.ReplaceAll(arg, v.Name, value) arg = strings.ReplaceAll(arg, "$"+v.Name, value) From 1e4011d5b194b57e28710a711f786c7df9e547c2 Mon Sep 17 00:00:00 2001 From: pb Date: Mon, 16 Jun 2025 18:22:58 +0200 Subject: [PATCH 3/9] refactored and added some values for dev purposes, modified the values set for path and key depending on the read or write condition --- workflow_builder/argo_builder.go | 123 ++++++++++++++++++++----------- 1 file changed, 82 insertions(+), 41 deletions(-) diff --git a/workflow_builder/argo_builder.go b/workflow_builder/argo_builder.go index 03b9d38..4bb0ae2 100644 --- a/workflow_builder/argo_builder.go +++ b/workflow_builder/argo_builder.go @@ -8,6 +8,7 @@ import ( "fmt" "oc-monitord/conf" . "oc-monitord/models" + tools2 "oc-monitord/tools" "os" "strings" @@ -17,7 +18,9 @@ import ( "cloud.o-forge.io/core/oc-lib/logs" "cloud.o-forge.io/core/oc-lib/models/common/enum" "cloud.o-forge.io/core/oc-lib/models/resources" + "cloud.o-forge.io/core/oc-lib/models/utils" w "cloud.o-forge.io/core/oc-lib/models/workflow" + "cloud.o-forge.io/core/oc-lib/models/workflow/graph" "github.com/nwtgck/go-fakelish" "github.com/rs/zerolog" "gopkg.in/yaml.v3" @@ -73,7 +76,7 @@ func (b *ArgoBuilder) CreateDAG(namespace string, write bool) ( int, []string, [ if b.Timeout > 0 { b.Workflow.Spec.Timeout = b.Timeout } - b.Workflow.Spec.ServiceAccountName = "sa-"+namespace + // b.Workflow.Spec.ServiceAccountName = "sa-"+namespace b.Workflow.Spec.Entrypoint = "dag" b.Workflow.ApiVersion = "argoproj.io/v1alpha1" b.Workflow.Kind = "Workflow" @@ -191,54 +194,23 @@ func (b *ArgoBuilder) createArgoTemplates(namespace string, template.Metadata.Labels = make(map[string]string) template.Metadata.Labels["app"] = "oc-service-" + processing.GetName() // Construct the template for the k8s service and add a link in graph between k8s service and processing } - related := b.OriginWorkflow.GetByRelatedProcessing(id, b.OriginWorkflow.Graph.IsStorage) + + related := b.OriginWorkflow.GetByRelatedProcessing(id, b.OriginWorkflow.Graph.IsStorage) // Retrieve all of the storage node linked to the processing for which we create the template for _, r := range related { storage := r.Node.(*resources.StorageResource) for _, linkToStorage := range r.Links { for _, rw := range linkToStorage.StorageLinkInfos { - art := Artifact{Path: template.ReplacePerEnv(rw.Source, linkToStorage.Env)} + var art Artifact + artifactBaseName := strings.Join(strings.Split(storage.GetName(), " "), "-") + "-" + strings.Replace(rw.FileName,".","-",-1) // Parameter/Artifact name must consist of alpha-numeric characters, '_' or '-' if rw.Write { - art.Name = storage.GetName() + "-" + rw.Destination + "-input-write" + art = Artifact{Path: template.ReplacePerEnv(rw.Source, linkToStorage.Env)} // When we are writing to the s3 the Path element is the path to the file in the pod + art.Name = artifactBaseName + "-input-write" } else { - art.Name = storage.GetName() + "-" + rw.Destination + "-input-read" + art = Artifact{Path: template.ReplacePerEnv(rw.Destination + "/" + rw.FileName, linkToStorage.Env)} // When we are reading from the s3 the Path element in pod should be the destination of the file + art.Name = artifactBaseName + "-input-read" } if storage.StorageType == enum.S3 { - art.S3 = &Key{ - Key: template.ReplacePerEnv(rw.Destination+"/"+rw.FileName, linkToStorage.Env), - Insecure: true, // temporary - } - sel := storage.GetSelectedInstance() - if sel != nil { - if sel.(*resources.StorageResourceInstance).Credentials != nil { - tool, err := tools2.NewService(conf.GetConfig().Mode) - if err != nil || tool == nil { - logger.Error().Msg("Could not create the access secret") - } else { - id, err := tool.CreateAccessSecret(namespace, - sel.(*resources.StorageResourceInstance).Credentials.Login, - sel.(*resources.StorageResourceInstance).Credentials.Pass) - if err == nil { - art.S3.AccessKeySecret = &Secret{ - Name: id, - Key: "access-key", - } - art.S3.SecretKeySecret = &Secret{ - Name: id, - Key: "secret-key", - } - } - } - } - art.S3.Key = strings.ReplaceAll(art.S3.Key, sel.(*resources.StorageResourceInstance).Source+"/", "") - art.S3.Key = strings.ReplaceAll(art.S3.Key, sel.(*resources.StorageResourceInstance).Source, "") - splits := strings.Split(art.S3.EndPoint, "/") - if len(splits) > 1 { - art.S3.Bucket = splits[0] - art.S3.EndPoint = strings.Join(splits[1:], "/") - } else { - art.S3.Bucket = splits[0] - } - } + b.addS3annotations(&art, template, rw, linkToStorage, storage, namespace) } if rw.Write { template.Outputs.Artifacts = append(template.Inputs.Artifacts, art) @@ -264,6 +236,75 @@ func (b *ArgoBuilder) createArgoTemplates(namespace string, return volumes, firstItems, lastItems } +func (b *ArgoBuilder) addS3annotations(art *Artifact, template *Template, rw graph.StorageProcessingGraphLink, linkToStorage graph.GraphLink, storage *resources.StorageResource, namespace string) { + art.S3 = &Key{ + // Key: template.ReplacePerEnv(rw.Destination+"/"+rw.FileName, linkToStorage.Env), + Insecure: true, // temporary + } + if rw.Write { + art.S3.Key = rw.Destination + "/" + rw.FileName + } else { + art.S3.Key = rw.Source + } + sel := storage.GetSelectedInstance() + // v0.1 : add the storage.Source to the s3 object + // v0.2 : test if the storage.Source exists in the configMap and quit if not + // v1 : v0.2 + if doesn't exist edit/create the configMap with the response from API call + if sel != nil { + b.addAuthInformation(sel, namespace, art) + art.S3.Bucket = "oc-bucket" // DEFAULT : will need to update this to create an unique + art.S3.EndPoint = sel.(*resources.StorageResourceInstance).Source + } +} + +func (*ArgoBuilder) addAuthInformation(sel utils.DBObject, namespace string, art *Artifact) { + if sel.(*resources.StorageResourceInstance).Credentials != nil { + tool, err := tools2.NewService(conf.GetConfig().Mode) + if err != nil || tool == nil { + logger.Error().Msg("Could not create the access secret") + } else { + id, err := tool.CreateAccessSecret(namespace, + sel.(*resources.StorageResourceInstance).Credentials.Login, + sel.(*resources.StorageResourceInstance).Credentials.Pass) + if err == nil { + art.S3.AccessKeySecret = &Secret{ + Name: id, + Key: "access-key", + } + art.S3.SecretKeySecret = &Secret{ + Name: id, + Key: "secret-key", + } + } + } + } + + // DELETE THIS AFTER FEAYURE IMPLEMENTATION + // CODE TO TEST THE BUCKET WITH SAME CREDENTIAL + + art.S3.AccessKeySecret = &Secret{ + Name: "argo-minio-secret", + Key: "accessKeySecret", + } + + art.S3.SecretKeySecret = &Secret{ + Name: "argo-minio-secret", + Key: "secretKeySecret", + } + // DELETE THIS AFTER FEAYURE IMPLEMENTATION + // CODE TO TEST THE BUCKET WITH SAME CREDENTIAL + + art.S3.Key = strings.ReplaceAll(art.S3.Key, sel.(*resources.StorageResourceInstance).Source+"/", "") + art.S3.Key = strings.ReplaceAll(art.S3.Key, sel.(*resources.StorageResourceInstance).Source, "") + splits := strings.Split(art.S3.EndPoint, "/") + if len(splits) > 1 { + art.S3.Bucket = splits[0] + art.S3.EndPoint = strings.Join(splits[1:], "/") + } else { + art.S3.Bucket = splits[0] + } +} + func (b *ArgoBuilder) addTaskToArgo(dag *Dag, graphItemID string, processing *resources.ProcessingResource, firstItems []string, lastItems []string) (*Dag, []string, []string) { unique_name := getArgoName(processing.GetName(), graphItemID) From a46708842bd9a3360a02a63ebea5275261f95d77 Mon Sep 17 00:00:00 2001 From: pb Date: Fri, 20 Jun 2025 11:28:12 +0200 Subject: [PATCH 4/9] added the methods to write the appropriate argo annotation when using a storage of type s3 as an artifact repository, with hardcoded credentials --- go.mod | 32 +++++------ go.sum | 72 +++++++++++++---------- main.go | 20 +++---- tools/interface.go | 4 +- tools/kubernetes.go | 35 ++++++++--- workflow_builder/argo_builder.go | 99 +++++++++++++++++++------------- 6 files changed, 156 insertions(+), 106 deletions(-) diff --git a/go.mod b/go.mod index 3e0e1b7..db32b2c 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.23.1 toolchain go1.23.3 require ( - cloud.o-forge.io/core/oc-lib v0.0.0-20250313155727-88c88cac5bc9 + cloud.o-forge.io/core/oc-lib v0.0.0-20250620085001-583ca2fbacd5 github.com/akamensky/argparse v1.4.0 github.com/google/uuid v1.6.0 github.com/goraz/onion v0.1.3 @@ -15,7 +15,7 @@ require ( ) require ( - github.com/beego/beego/v2 v2.3.7 // indirect + github.com/beego/beego/v2 v2.3.8 // indirect github.com/go-playground/validator/v10 v10.26.0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect @@ -35,7 +35,7 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.8 // indirect + github.com/gabriel-vasile/mimetype v1.4.9 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.20.4 // indirect @@ -60,14 +60,14 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/montanaflynn/stats v0.7.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/nats-io/nats.go v1.41.0 // indirect - github.com/nats-io/nkeys v0.4.10 // indirect + github.com/nats-io/nats.go v1.43.0 // indirect + github.com/nats-io/nkeys v0.4.11 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.22.0 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.63.0 // indirect - github.com/prometheus/procfs v0.16.0 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.64.0 // indirect + github.com/prometheus/procfs v0.16.1 // indirect github.com/robfig/cron v1.2.0 // indirect github.com/shiena/ansicolor v0.0.0-20230509054315-a9deabde6e02 // indirect github.com/smartystreets/goconvey v1.6.4 // indirect @@ -76,14 +76,14 @@ require ( github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect - go.mongodb.org/mongo-driver v1.17.3 // indirect - golang.org/x/crypto v0.37.0 // indirect - golang.org/x/net v0.39.0 // indirect - golang.org/x/oauth2 v0.25.0 // indirect - golang.org/x/sync v0.13.0 // indirect - golang.org/x/sys v0.32.0 // indirect - golang.org/x/term v0.31.0 // indirect - golang.org/x/text v0.24.0 // indirect + go.mongodb.org/mongo-driver v1.17.4 // indirect + golang.org/x/crypto v0.39.0 // indirect + golang.org/x/net v0.41.0 // indirect + golang.org/x/oauth2 v0.30.0 // indirect + golang.org/x/sync v0.15.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/term v0.32.0 // indirect + golang.org/x/text v0.26.0 // indirect golang.org/x/time v0.7.0 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect diff --git a/go.sum b/go.sum index 4ad81c9..7d8d3ec 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,9 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.o-forge.io/core/oc-lib v0.0.0-20250313155727-88c88cac5bc9 h1:mSFFPwil5Ih+RPBvn88MBerQMtsoHnOuyCZQaf91a34= -cloud.o-forge.io/core/oc-lib v0.0.0-20250313155727-88c88cac5bc9/go.mod h1:2roQbUpv3a6mTIr5oU1ux31WbN8YucyyQvCQ0FqwbcE= +cloud.o-forge.io/core/oc-lib v0.0.0-20250604083300-387785b40cb0 h1:iEm/Rf9I0OSCcncuFy61YOSZ3jdRlhJ/oLD97Pc2pCQ= +cloud.o-forge.io/core/oc-lib v0.0.0-20250604083300-387785b40cb0/go.mod h1:2roQbUpv3a6mTIr5oU1ux31WbN8YucyyQvCQ0FqwbcE= +cloud.o-forge.io/core/oc-lib v0.0.0-20250620085001-583ca2fbacd5 h1:FEBwueVOOWKYf0tJuE0EKNIbjxmTyCMgkT4qATYsfbo= +cloud.o-forge.io/core/oc-lib v0.0.0-20250620085001-583ca2fbacd5/go.mod h1:vHWauJsS6ryf7UDqq8hRXoYD5RsONxcFTxeZPOztEuI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/akamensky/argparse v1.4.0 h1:YGzvsTqCvbEZhL8zZu2AiA5nq805NZh75JNj4ajn1xc= github.com/akamensky/argparse v1.4.0/go.mod h1:S5kwC7IuDcEr5VeXtGPRVZ5o/FdhcMlQz4IZQuw64xA= @@ -9,8 +11,8 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/argoproj/argo-workflows/v3 v3.6.4 h1:5+Cc1UwaQE5ka3w7R3hxZ1TK3M6VjDEXA5WSQ/IXrxY= github.com/argoproj/argo-workflows/v3 v3.6.4/go.mod h1:2f5zB8CkbNCCO1od+kd1dWkVokqcuyvu+tc+Jwx1MZg= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/beego/beego/v2 v2.3.7 h1:z4btKtjU/rfp5BiYHkGD2QPjK9i1E9GH+I7vfhn6Agk= -github.com/beego/beego/v2 v2.3.7/go.mod h1:5cqHsOHJIxkq44tBpRvtDe59GuVRVv/9/tyVDxd5ce4= +github.com/beego/beego/v2 v2.3.8 h1:wplhB1pF4TxR+2SS4PUej8eDoH4xGfxuHfS7wAk9VBc= +github.com/beego/beego/v2 v2.3.8/go.mod h1:8vl9+RrXqvodrl9C8yivX1e6le6deCK6RWeq8R7gTTg= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/biter777/countries v1.7.5 h1:MJ+n3+rSxWQdqVJU8eBy9RqcdH6ePPn4PJHocVWUa+Q= @@ -43,8 +45,8 @@ github.com/etcd-io/etcd v3.3.17+incompatible/go.mod h1:cdZ77EstHBwVtD6iTgzgvogwc github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= -github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= -github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= +github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= +github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -147,10 +149,10 @@ github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8 github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nats-io/nats.go v1.41.0 h1:PzxEva7fflkd+n87OtQTXqCTyLfIIMFJBpyccHLE2Ko= -github.com/nats-io/nats.go v1.41.0/go.mod h1:wV73x0FSI/orHPSYoyMeJB+KajMDoWyXmFaRrrYaaTo= -github.com/nats-io/nkeys v0.4.10 h1:glmRrpCmYLHByYcePvnTBEAwawwapjCPMjy2huw20wc= -github.com/nats-io/nkeys v0.4.10/go.mod h1:OjRrnIKnWBFl+s4YK5ChQfvHP2fxqZexrKJoVVyWB3U= +github.com/nats-io/nats.go v1.43.0 h1:uRFZ2FEoRvP64+UUhaTokyS18XBCR/xM2vQZKO4i8ug= +github.com/nats-io/nats.go v1.43.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= +github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0= +github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nwtgck/go-fakelish v0.1.3 h1:bA8/xa9hQmzppexIhBvdmztcd/PJ4SPuAUTBdMKZ8G4= @@ -171,12 +173,12 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= -github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= -github.com/prometheus/procfs v0.16.0 h1:xh6oHhKwnOJKMYiYBDWmkHqQPyiY40sny36Cmx2bbsM= -github.com/prometheus/procfs v0.16.0/go.mod h1:8veyXUu3nGP7oaCxhX6yeaM5u4stL2FeMXnCqhDthZg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQPGO4= +github.com/prometheus/common v0.64.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -230,14 +232,18 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mongodb.org/mongo-driver v1.17.3 h1:TQyXhnsWfWtgAhMtOgtYHMTkZIfBTpMTsMnd9ZBeHxQ= go.mongodb.org/mongo-driver v1.17.3/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= +go.mongodb.org/mongo-driver v1.17.4 h1:jUorfmVzljjr0FLzYQsGP8cgN/qzzxlY9Vh0C9KFXVw= +go.mongodb.org/mongo-driver v1.17.4/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= -golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= +golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -257,12 +263,14 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= -golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= +golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= -golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -270,8 +278,10 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= -golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= +golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -287,18 +297,20 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= -golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= +golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= +golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/main.go b/main.go index 721a23b..2e029ff 100644 --- a/main.go +++ b/main.go @@ -102,16 +102,16 @@ func main() { workflowName = getContainerName(argoFilePath) - if conf.GetConfig().KubeHost == "" { - // Not in a k8s environment, get conf from parameters - logger.Info().Msg("Executes outside of k8s") - executeOutside(argoFilePath, builder.Workflow) - } else { - // Executed in a k8s environment - logger.Info().Msg("Executes inside a k8s") - // executeInside(exec.GetID(), "argo", argo_file_path, stepMax) // commenting to use conf.ExecutionID instead of exec.GetID() - executeInside(conf.GetConfig().ExecutionID, exec.ExecutionsID, argoFilePath) - } + // if conf.GetConfig().KubeHost == "" { + // // Not in a k8s environment, get conf from parameters + // logger.Info().Msg("Executes outside of k8s") + // executeOutside(argoFilePath, builder.Workflow) + // } else { + // // Executed in a k8s environment + // logger.Info().Msg("Executes inside a k8s") + // // executeInside(exec.GetID(), "argo", argo_file_path, stepMax) // commenting to use conf.ExecutionID instead of exec.GetID() + // executeInside(conf.GetConfig().ExecutionID, exec.ExecutionsID, argoFilePath) + // } } // So far we only log the output from diff --git a/tools/interface.go b/tools/interface.go index b6f2610..9b38a2e 100644 --- a/tools/interface.go +++ b/tools/interface.go @@ -4,14 +4,16 @@ import ( "errors" "io" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/watch" ) type Tool interface { CreateArgoWorkflow(path string, ns string) (string, error) - CreateAccessSecret(ns string, login string, password string) (string, error) + CreateAccessSecret(user string, password string, storageId string, namespace string) (string, error) GetArgoWatch(executionId string, wfName string) (watch.Interface, error) GetPodLogger(ns string, wfName string, podName string) (io.ReadCloser, error) + GetS3Secret(storageId string, namespace string) *v1.Secret } var _service = map[string]func() (Tool, error){ diff --git a/tools/kubernetes.go b/tools/kubernetes.go index ae7fe14..c45d4da 100644 --- a/tools/kubernetes.go +++ b/tools/kubernetes.go @@ -2,7 +2,6 @@ package tools import ( "context" - "encoding/base64" "errors" "fmt" "io" @@ -13,8 +12,8 @@ import ( wfv1 "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1" "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned" - "github.com/google/uuid" v1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -31,7 +30,7 @@ type KubernetesTools struct { func NewKubernetesTool() (Tool, error) { // Load Kubernetes config (from ~/.kube/config) config := &rest.Config{ - Host: conf.GetConfig().KubeHost + ":" + conf.GetConfig().KubePort, + Host: "https://" + conf.GetConfig().KubeHost + ":" + conf.GetConfig().KubePort, TLSClientConfig: rest.TLSClientConfig{ CAData: []byte(conf.GetConfig().KubeCA), CertData: []byte(conf.GetConfig().KubeCert), @@ -88,21 +87,20 @@ func (k *KubernetesTools) CreateArgoWorkflow(path string, ns string) (string, er return createdWf.Name, nil } -func (k *KubernetesTools) CreateAccessSecret(ns string, login string, password string) (string, error) { +func (k *KubernetesTools) CreateAccessSecret(access string, password string, storageId string, namespace string) (string, error) { // Namespace where the secret will be created - namespace := "default" // Encode the secret data (Kubernetes requires base64-encoded values) secretData := map[string][]byte{ - "access-key": []byte(base64.StdEncoding.EncodeToString([]byte(login))), - "secret-key": []byte(base64.StdEncoding.EncodeToString([]byte(password))), + "access-key": []byte(access), + "secret-key": []byte(password), } // Define the Secret object - name := uuid.New().String() + name := storageId+"-secret-s3" secret := &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: name, - Namespace: ns, + Namespace: namespace, }, Type: v1.SecretTypeOpaque, Data: secretData, @@ -112,9 +110,28 @@ func (k *KubernetesTools) CreateAccessSecret(ns string, login string, password s if err != nil { return "", errors.New("Error creating secret: " + err.Error()) } + return name, nil } +func (k *KubernetesTools) GetS3Secret(storageId string, namespace string) *v1.Secret { + + secret, err := k.Set.CoreV1().Secrets(namespace).Get(context.TODO(), storageId + "-secret-s3", metav1.GetOptions{}) + // Get(context.TODO(),storageId + "-artifact-server", metav1.GetOptions{}) + + if err != nil && !k8serrors.IsNotFound(err) { + l := utils.GetLogger() + l.Fatal().Msg("An error happened when retrieving secret in " + namespace + " : " + err.Error()) + } + if k8serrors.IsNotFound(err) { + return nil + } + + return secret + // return secret +} + + func (k *KubernetesTools) GetArgoWatch(executionId string, wfName string) (watch.Interface, error){ options := metav1.ListOptions{FieldSelector: "metadata.name=oc-monitor-"+wfName} diff --git a/workflow_builder/argo_builder.go b/workflow_builder/argo_builder.go index 4bb0ae2..11bc224 100644 --- a/workflow_builder/argo_builder.go +++ b/workflow_builder/argo_builder.go @@ -18,7 +18,6 @@ import ( "cloud.o-forge.io/core/oc-lib/logs" "cloud.o-forge.io/core/oc-lib/models/common/enum" "cloud.o-forge.io/core/oc-lib/models/resources" - "cloud.o-forge.io/core/oc-lib/models/utils" w "cloud.o-forge.io/core/oc-lib/models/workflow" "cloud.o-forge.io/core/oc-lib/models/workflow/graph" "github.com/nwtgck/go-fakelish" @@ -195,23 +194,32 @@ func (b *ArgoBuilder) createArgoTemplates(namespace string, template.Metadata.Labels["app"] = "oc-service-" + processing.GetName() // Construct the template for the k8s service and add a link in graph between k8s service and processing } + volumes = b.addStorageAnnotations(id, template, namespace, volumes) + b.Workflow.Spec.Templates = append(b.Workflow.Spec.Templates, *template) + return volumes, firstItems, lastItems +} + +func (b *ArgoBuilder) addStorageAnnotations(id string, template *Template, namespace string, volumes []VolumeMount) []VolumeMount { related := b.OriginWorkflow.GetByRelatedProcessing(id, b.OriginWorkflow.Graph.IsStorage) // Retrieve all of the storage node linked to the processing for which we create the template for _, r := range related { storage := r.Node.(*resources.StorageResource) for _, linkToStorage := range r.Links { for _, rw := range linkToStorage.StorageLinkInfos { var art Artifact - artifactBaseName := strings.Join(strings.Split(storage.GetName(), " "), "-") + "-" + strings.Replace(rw.FileName,".","-",-1) // Parameter/Artifact name must consist of alpha-numeric characters, '_' or '-' + artifactBaseName := strings.Join(strings.Split(storage.GetName(), " "), "-") + "-" + strings.Replace(rw.FileName, ".", "-", -1) // Parameter/Artifact name must consist of alpha-numeric characters, '_' or '-' if rw.Write { - art = Artifact{Path: template.ReplacePerEnv(rw.Source, linkToStorage.Env)} // When we are writing to the s3 the Path element is the path to the file in the pod + art = Artifact{Path: template.ReplacePerEnv(rw.Source, linkToStorage.Env)} // When we are writing to the s3 the Path element is the path to the file in the pod art.Name = artifactBaseName + "-input-write" } else { - art = Artifact{Path: template.ReplacePerEnv(rw.Destination + "/" + rw.FileName, linkToStorage.Env)} // When we are reading from the s3 the Path element in pod should be the destination of the file + art = Artifact{Path: template.ReplacePerEnv(rw.Destination+"/"+rw.FileName, linkToStorage.Env)} // When we are reading from the s3 the Path element in pod should be the destination of the file art.Name = artifactBaseName + "-input-read" } + if storage.StorageType == enum.S3 { + b.addS3annotations(&art, template, rw, linkToStorage, storage, namespace) } + if rw.Write { template.Outputs.Artifacts = append(template.Inputs.Artifacts, art) } else { @@ -232,11 +240,11 @@ func (b *ArgoBuilder) createArgoTemplates(namespace string, }, volumes) } } - b.Workflow.Spec.Templates = append(b.Workflow.Spec.Templates, *template) - return volumes, firstItems, lastItems -} + return volumes +} func (b *ArgoBuilder) addS3annotations(art *Artifact, template *Template, rw graph.StorageProcessingGraphLink, linkToStorage graph.GraphLink, storage *resources.StorageResource, namespace string) { + art.S3 = &Key{ // Key: template.ReplacePerEnv(rw.Destination+"/"+rw.FileName, linkToStorage.Env), Insecure: true, // temporary @@ -251,48 +259,57 @@ func (b *ArgoBuilder) addS3annotations(art *Artifact, template *Template, rw gra // v0.2 : test if the storage.Source exists in the configMap and quit if not // v1 : v0.2 + if doesn't exist edit/create the configMap with the response from API call if sel != nil { - b.addAuthInformation(sel, namespace, art) - art.S3.Bucket = "oc-bucket" // DEFAULT : will need to update this to create an unique + b.addAuthInformation(storage, namespace, art) + art.S3.Bucket = "oc-bucket" // DEFAULT : will need to update this to create an unique art.S3.EndPoint = sel.(*resources.StorageResourceInstance).Source } } -func (*ArgoBuilder) addAuthInformation(sel utils.DBObject, namespace string, art *Artifact) { - if sel.(*resources.StorageResourceInstance).Credentials != nil { - tool, err := tools2.NewService(conf.GetConfig().Mode) - if err != nil || tool == nil { - logger.Error().Msg("Could not create the access secret") - } else { - id, err := tool.CreateAccessSecret(namespace, - sel.(*resources.StorageResourceInstance).Credentials.Login, - sel.(*resources.StorageResourceInstance).Credentials.Pass) - if err == nil { - art.S3.AccessKeySecret = &Secret{ - Name: id, - Key: "access-key", - } - art.S3.SecretKeySecret = &Secret{ - Name: id, - Key: "secret-key", - } - } +func (b *ArgoBuilder) SetupS3Credentials(storage *resources.StorageResource, namespace string, tool tools2.Tool) (string, error) { + s := tool.GetS3Secret(storage.UUID, namespace) + // var s *v1.Secret + + if s == nil { + id, err := tool.CreateAccessSecret( + "hF9wRGog75JuMdshWeEZ", + "OwXXJkVQyb5l1aVPdOegKOtDJGoP1dJYeo8O7mDW", + storage.UUID, + namespace, + ) + if err != nil { + l := oclib.GetLogger() + l.Fatal().Msg("Error when creating the secret holding credentials for S3 access in " + namespace + " : " + err.Error()) } + + return id, nil } - // DELETE THIS AFTER FEAYURE IMPLEMENTATION - // CODE TO TEST THE BUCKET WITH SAME CREDENTIAL + // s.Name = "toto" + return s.Name, nil + +} - art.S3.AccessKeySecret = &Secret{ - Name: "argo-minio-secret", - Key: "accessKeySecret", - } +func (b *ArgoBuilder) addAuthInformation(storage *resources.StorageResource, namespace string, art *Artifact) { + + sel := storage.GetSelectedInstance() + + tool, err := tools2.NewService(conf.GetConfig().Mode) + if err != nil || tool == nil { + logger.Fatal().Msg("Could not create the access secret :" + err.Error()) + } - art.S3.SecretKeySecret = &Secret{ - Name: "argo-minio-secret", - Key: "secretKeySecret", - } - // DELETE THIS AFTER FEAYURE IMPLEMENTATION - // CODE TO TEST THE BUCKET WITH SAME CREDENTIAL + secretName, err := b.SetupS3Credentials(storage, namespace, tool) // this method return should be updated once we have decided how to retrieve credentials + + if err == nil { + art.S3.AccessKeySecret = &Secret{ + Name: secretName, + Key: "access-key", + } + art.S3.SecretKeySecret = &Secret{ + Name: secretName, + Key: "secret-key", + } + } art.S3.Key = strings.ReplaceAll(art.S3.Key, sel.(*resources.StorageResourceInstance).Source+"/", "") art.S3.Key = strings.ReplaceAll(art.S3.Key, sel.(*resources.StorageResourceInstance).Source, "") @@ -305,6 +322,8 @@ func (*ArgoBuilder) addAuthInformation(sel utils.DBObject, namespace string, art } } + + func (b *ArgoBuilder) addTaskToArgo(dag *Dag, graphItemID string, processing *resources.ProcessingResource, firstItems []string, lastItems []string) (*Dag, []string, []string) { unique_name := getArgoName(processing.GetName(), graphItemID) From 722a42e36dfb9e877f749d98955c031f05e3521a Mon Sep 17 00:00:00 2001 From: pb Date: Thu, 10 Jul 2025 17:31:38 +0200 Subject: [PATCH 5/9] removed the returned value from the callRemoteExecution method, all the resp handling is done inside it --- workflow_builder/argo_builder.go | 109 ++++++++++++++++--------------- 1 file changed, 57 insertions(+), 52 deletions(-) diff --git a/workflow_builder/argo_builder.go b/workflow_builder/argo_builder.go index 11bc224..3fbf04f 100644 --- a/workflow_builder/argo_builder.go +++ b/workflow_builder/argo_builder.go @@ -265,14 +265,49 @@ func (b *ArgoBuilder) addS3annotations(art *Artifact, template *Template, rw gra } } + +func (b *ArgoBuilder) addAuthInformation(storage *resources.StorageResource, namespace string, art *Artifact) { + + sel := storage.GetSelectedInstance() + + tool, err := tools2.NewService(conf.GetConfig().Mode) + if err != nil || tool == nil { + logger.Fatal().Msg("Could not create the access secret :" + err.Error()) + } + + secretName, err := b.SetupS3Credentials(storage, namespace, tool) // this method return should be updated once we have decided how to retrieve credentials + + if err == nil { + art.S3.AccessKeySecret = &Secret{ + Name: secretName, + Key: "access-key", + } + art.S3.SecretKeySecret = &Secret{ + Name: secretName, + Key: "secret-key", + } + } + + art.S3.Key = strings.ReplaceAll(art.S3.Key, sel.(*resources.StorageResourceInstance).Source+"/", "") + art.S3.Key = strings.ReplaceAll(art.S3.Key, sel.(*resources.StorageResourceInstance).Source, "") + splits := strings.Split(art.S3.EndPoint, "/") + if len(splits) > 1 { + art.S3.Bucket = splits[0] + art.S3.EndPoint = strings.Join(splits[1:], "/") + } else { + art.S3.Bucket = splits[0] + } +} + func (b *ArgoBuilder) SetupS3Credentials(storage *resources.StorageResource, namespace string, tool tools2.Tool) (string, error) { s := tool.GetS3Secret(storage.UUID, namespace) // var s *v1.Secret - + accessKey, secretKey := retrieveMinioCredential("peer",namespace) + if s == nil { id, err := tool.CreateAccessSecret( - "hF9wRGog75JuMdshWeEZ", - "OwXXJkVQyb5l1aVPdOegKOtDJGoP1dJYeo8O7mDW", + accessKey, + secretKey, storage.UUID, namespace, ) @@ -289,45 +324,15 @@ func (b *ArgoBuilder) SetupS3Credentials(storage *resources.StorageResource, nam } -func (b *ArgoBuilder) addAuthInformation(storage *resources.StorageResource, namespace string, art *Artifact) { - - sel := storage.GetSelectedInstance() - - tool, err := tools2.NewService(conf.GetConfig().Mode) - if err != nil || tool == nil { - logger.Fatal().Msg("Could not create the access secret :" + err.Error()) - } - - secretName, err := b.SetupS3Credentials(storage, namespace, tool) // this method return should be updated once we have decided how to retrieve credentials - - if err == nil { - art.S3.AccessKeySecret = &Secret{ - Name: secretName, - Key: "access-key", - } - art.S3.SecretKeySecret = &Secret{ - Name: secretName, - Key: "secret-key", - } - } - - art.S3.Key = strings.ReplaceAll(art.S3.Key, sel.(*resources.StorageResourceInstance).Source+"/", "") - art.S3.Key = strings.ReplaceAll(art.S3.Key, sel.(*resources.StorageResourceInstance).Source, "") - splits := strings.Split(art.S3.EndPoint, "/") - if len(splits) > 1 { - art.S3.Bucket = splits[0] - art.S3.EndPoint = strings.Join(splits[1:], "/") - } else { - art.S3.Bucket = splits[0] - } +// This method needs to evolve to an API call to the peer passed as a parameter +func retrieveMinioCredential(peer string, namespace string) (string,string) { + return "hF9wRGog75JuMdshWeEZ", "OwXXJkVQyb5l1aVPdOegKOtDJGoP1dJYeo8O7mDW" } - - func (b *ArgoBuilder) addTaskToArgo(dag *Dag, graphItemID string, processing *resources.ProcessingResource, firstItems []string, lastItems []string) (*Dag, []string, []string) { - unique_name := getArgoName(processing.GetName(), graphItemID) - step := Task{Name: unique_name, Template: unique_name} + unique_name := getArgoName(processing.GetName(), graphItemID) + step := Task{Name: unique_name, Template: unique_name} instance := processing.GetSelectedInstance() if instance != nil { for _, value := range instance.(*resources.ProcessingInstance).Env { @@ -485,21 +490,21 @@ func (b *ArgoBuilder) retrieveProcessingCompute(graphID string) *resources.Compu // Execute the last actions once the YAML file for the Argo Workflow is created func (b *ArgoBuilder) CompleteBuild(executionsId string) (string, error) { logger.Info().Msg(fmt.Sprint("DEV :: Completing build")) - setter := AdmiraltySetter{Id: executionsId} - // Setup admiralty for each node - for _, peer := range b.RemotePeers { - logger.Info().Msg(fmt.Sprint("DEV :: Launching Admiralty Setup for ", peer)) - setter.InitializeAdmiralty(conf.GetConfig().PeerID,peer) - } + // setter := AdmiraltySetter{Id: executionsId} + // // Setup admiralty for each node + // for _, peer := range b.RemotePeers { + // logger.Info().Msg(fmt.Sprint("DEV :: Launching Admiralty Setup for ", peer)) + // setter.InitializeAdmiralty(conf.GetConfig().PeerID,peer) + // } - // Update the name of the admiralty node to use - for _, template := range b.Workflow.Spec.Templates { - if len(template.Metadata.Annotations) > 0 { - if peerId, ok := template.Metadata.Annotations["multicluster.admiralty.io/clustername"]; ok { - template.Metadata.Annotations["multicluster.admiralty.io/clustername"] = "target-" + peerId + "-" + conf.GetConfig().ExecutionID - } - } - } + // // Update the name of the admiralty node to use + // for _, template := range b.Workflow.Spec.Templates { + // if len(template.Metadata.Annotations) > 0 { + // if peerId, ok := template.Metadata.Annotations["multicluster.admiralty.io/clustername"]; ok { + // template.Metadata.Annotations["multicluster.admiralty.io/clustername"] = "target-" + peerId + "-" + conf.GetConfig().ExecutionID + // } + // } + // } // Generate the YAML file random_name := fakelish.GenerateFakeWord(5, 8) + "-" + fakelish.GenerateFakeWord(5, 8) From a50ede2688d1438ffb6e96ba2d85e658c9a46e3b Mon Sep 17 00:00:00 2001 From: pb Date: Fri, 11 Jul 2025 10:24:52 +0200 Subject: [PATCH 6/9] forgot go files --- go.mod | 8 ++++---- go.sum | 8 ++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index db32b2c..9d3a443 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.23.1 toolchain go1.23.3 require ( - cloud.o-forge.io/core/oc-lib v0.0.0-20250620085001-583ca2fbacd5 + cloud.o-forge.io/core/oc-lib v0.0.0-20250704084459-443546027b27 github.com/akamensky/argparse v1.4.0 github.com/google/uuid v1.6.0 github.com/goraz/onion v0.1.3 @@ -16,7 +16,7 @@ require ( require ( github.com/beego/beego/v2 v2.3.8 // indirect - github.com/go-playground/validator/v10 v10.26.0 // indirect + github.com/go-playground/validator/v10 v10.27.0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect @@ -66,8 +66,8 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.22.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.64.0 // indirect - github.com/prometheus/procfs v0.16.1 // indirect + github.com/prometheus/common v0.65.0 // indirect + github.com/prometheus/procfs v0.17.0 // indirect github.com/robfig/cron v1.2.0 // indirect github.com/shiena/ansicolor v0.0.0-20230509054315-a9deabde6e02 // indirect github.com/smartystreets/goconvey v1.6.4 // indirect diff --git a/go.sum b/go.sum index 7d8d3ec..b6c6808 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ cloud.o-forge.io/core/oc-lib v0.0.0-20250604083300-387785b40cb0 h1:iEm/Rf9I0OSCc cloud.o-forge.io/core/oc-lib v0.0.0-20250604083300-387785b40cb0/go.mod h1:2roQbUpv3a6mTIr5oU1ux31WbN8YucyyQvCQ0FqwbcE= cloud.o-forge.io/core/oc-lib v0.0.0-20250620085001-583ca2fbacd5 h1:FEBwueVOOWKYf0tJuE0EKNIbjxmTyCMgkT4qATYsfbo= cloud.o-forge.io/core/oc-lib v0.0.0-20250620085001-583ca2fbacd5/go.mod h1:vHWauJsS6ryf7UDqq8hRXoYD5RsONxcFTxeZPOztEuI= +cloud.o-forge.io/core/oc-lib v0.0.0-20250704084459-443546027b27 h1:iogk6pV3gybzQDBXMI6Qd/jvSA1h+3oRE+vLl1MRjew= +cloud.o-forge.io/core/oc-lib v0.0.0-20250704084459-443546027b27/go.mod h1:vHWauJsS6ryf7UDqq8hRXoYD5RsONxcFTxeZPOztEuI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/akamensky/argparse v1.4.0 h1:YGzvsTqCvbEZhL8zZu2AiA5nq805NZh75JNj4ajn1xc= github.com/akamensky/argparse v1.4.0/go.mod h1:S5kwC7IuDcEr5VeXtGPRVZ5o/FdhcMlQz4IZQuw64xA= @@ -64,6 +66,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= +github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4= +github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -177,8 +181,12 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.64.0 h1:pdZeA+g617P7oGv1CzdTzyeShxAGrTBsolKNOLQPGO4= github.com/prometheus/common v0.64.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= +github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE= +github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= +github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= From 0b652f5dfc1e4d163463e012f5d585d9c7403583 Mon Sep 17 00:00:00 2001 From: pb Date: Fri, 11 Jul 2025 10:27:44 +0200 Subject: [PATCH 7/9] removed the returned value from the callRemoteExecution method, all the resp handling is done inside it --- workflow_builder/admiralty_setter.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/workflow_builder/admiralty_setter.go b/workflow_builder/admiralty_setter.go index b36cd72..edc897c 100644 --- a/workflow_builder/admiralty_setter.go +++ b/workflow_builder/admiralty_setter.go @@ -59,13 +59,13 @@ func (s *AdmiraltySetter) InitializeAdmiralty(localPeerID string,remotePeerID st ) logger.Info().Msg("\n\n Creating the Admiralty Source on " + remotePeerID + " ns-" + s.Id) - _ = s.callRemoteExecution(remotePeer, []int{http.StatusCreated, http.StatusConflict},caller, s.Id, tools.ADMIRALTY_SOURCE, tools.POST, nil, true) + s.callRemoteExecution(remotePeer, []int{http.StatusCreated, http.StatusConflict},caller, s.Id, tools.ADMIRALTY_SOURCE, tools.POST, nil, true) logger.Info().Msg("\n\n Retrieving kubeconfig with the secret on " + remotePeerID + " ns-" + s.Id) kubeconfig := s.getKubeconfig(remotePeer, caller) logger.Info().Msg("\n\n Creating a secret from the kubeconfig " + localPeerID + " ns-" + s.Id) - _ = s.callRemoteExecution(localPeer, []int{http.StatusCreated}, caller,s.Id, tools.ADMIRALTY_SECRET, tools.POST,kubeconfig, true) + s.callRemoteExecution(localPeer, []int{http.StatusCreated}, caller,s.Id, tools.ADMIRALTY_SECRET, tools.POST,kubeconfig, true) logger.Info().Msg("\n\n Creating the Admiralty Target on " + localPeerID + " in namespace " + s.Id ) - _ = s.callRemoteExecution(localPeer,[]int{http.StatusCreated, http.StatusConflict},caller,s.Id,tools.ADMIRALTY_TARGET,tools.POST, nil, true) + s.callRemoteExecution(localPeer,[]int{http.StatusCreated, http.StatusConflict},caller,s.Id,tools.ADMIRALTY_TARGET,tools.POST, nil, true) logger.Info().Msg("\n\n Checking for the creation of the admiralty node on " + localPeerID + " ns-" + s.Id) s.checkNodeStatus(localPeer,caller) @@ -74,7 +74,7 @@ func (s *AdmiraltySetter) InitializeAdmiralty(localPeerID string,remotePeerID st func (s *AdmiraltySetter) getKubeconfig(peer *peer.Peer, caller *tools.HTTPCaller) map[string]string { var kubedata map[string]string - _ = s.callRemoteExecution(peer, []int{http.StatusOK}, caller, s.Id, tools.ADMIRALTY_KUBECONFIG, tools.GET, nil, true) + s.callRemoteExecution(peer, []int{http.StatusOK}, caller, s.Id, tools.ADMIRALTY_KUBECONFIG, tools.GET, nil, true) if caller.LastResults["body"] == nil || len(caller.LastResults["body"].([]byte)) == 0 { l := utils.GetLogger() l.Error().Msg("Something went wrong when retrieving data from Get call for kubeconfig") @@ -90,9 +90,9 @@ func (s *AdmiraltySetter) getKubeconfig(peer *peer.Peer, caller *tools.HTTPCalle return kubedata } -func (*AdmiraltySetter) callRemoteExecution(peer *peer.Peer, expectedCode []int,caller *tools.HTTPCaller, dataID string, dt tools.DataType, method tools.METHOD, body interface{}, panicCode bool) *peer.PeerExecution { +func (*AdmiraltySetter) callRemoteExecution(peer *peer.Peer, expectedCode []int,caller *tools.HTTPCaller, dataID string, dt tools.DataType, method tools.METHOD, body interface{}, panicCode bool) { l := utils.GetLogger() - resp, err := peer.LaunchPeerExecution(peer.UUID, dataID, dt, method, body, caller) + _, err := peer.LaunchPeerExecution(peer.UUID, dataID, dt, method, body, caller) if err != nil { l.Error().Msg("Error when executing on peer at" + peer.Url) l.Error().Msg(err.Error()) @@ -109,7 +109,6 @@ func (*AdmiraltySetter) callRemoteExecution(peer *peer.Peer, expectedCode []int, } } - return resp } func (s *AdmiraltySetter) storeNodeName(caller *tools.HTTPCaller){ @@ -132,7 +131,7 @@ func (s *AdmiraltySetter) storeNodeName(caller *tools.HTTPCaller){ func (s *AdmiraltySetter) checkNodeStatus(localPeer *peer.Peer, caller *tools.HTTPCaller){ for i := range(5) { time.Sleep(10 * time.Second) // let some time for kube to generate the node - _ = s.callRemoteExecution(localPeer,[]int{http.StatusOK},caller,s.Id,tools.ADMIRALTY_NODES,tools.GET, nil, false) + s.callRemoteExecution(localPeer,[]int{http.StatusOK},caller,s.Id,tools.ADMIRALTY_NODES,tools.GET, nil, false) if caller.LastResults["code"] == 200 { s.storeNodeName(caller) return From 6bacf44f2f7dd47c88a99c3fdb45d8727b8e7ce6 Mon Sep 17 00:00:00 2001 From: pb Date: Mon, 28 Jul 2025 15:52:27 +0200 Subject: [PATCH 8/9] Updated argo build to use minio --- go.mod | 14 ++++++------- go.sum | 14 +++++++++++++ main.go | 20 +++++++++--------- workflow_builder/argo_builder.go | 36 ++++++++++++++++---------------- 4 files changed, 49 insertions(+), 35 deletions(-) diff --git a/go.mod b/go.mod index 9d3a443..1310908 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.23.1 toolchain go1.23.3 require ( - cloud.o-forge.io/core/oc-lib v0.0.0-20250704084459-443546027b27 + cloud.o-forge.io/core/oc-lib v0.0.0-20250715125819-e735f78e58c6 github.com/akamensky/argparse v1.4.0 github.com/google/uuid v1.6.0 github.com/goraz/onion v0.1.3 @@ -77,13 +77,13 @@ require ( github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect go.mongodb.org/mongo-driver v1.17.4 // indirect - golang.org/x/crypto v0.39.0 // indirect - golang.org/x/net v0.41.0 // indirect + golang.org/x/crypto v0.40.0 // indirect + golang.org/x/net v0.42.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.15.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/term v0.32.0 // indirect - golang.org/x/text v0.26.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/term v0.33.0 // indirect + golang.org/x/text v0.27.0 // indirect golang.org/x/time v0.7.0 // indirect google.golang.org/protobuf v1.36.6 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect diff --git a/go.sum b/go.sum index b6c6808..86e9972 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ cloud.o-forge.io/core/oc-lib v0.0.0-20250620085001-583ca2fbacd5 h1:FEBwueVOOWKYf cloud.o-forge.io/core/oc-lib v0.0.0-20250620085001-583ca2fbacd5/go.mod h1:vHWauJsS6ryf7UDqq8hRXoYD5RsONxcFTxeZPOztEuI= cloud.o-forge.io/core/oc-lib v0.0.0-20250704084459-443546027b27 h1:iogk6pV3gybzQDBXMI6Qd/jvSA1h+3oRE+vLl1MRjew= cloud.o-forge.io/core/oc-lib v0.0.0-20250704084459-443546027b27/go.mod h1:vHWauJsS6ryf7UDqq8hRXoYD5RsONxcFTxeZPOztEuI= +cloud.o-forge.io/core/oc-lib v0.0.0-20250715125819-e735f78e58c6 h1:Gnkv59Ntl2RebC5tNauXuxyRXLfZ2XAJ0+ujMyFte5U= +cloud.o-forge.io/core/oc-lib v0.0.0-20250715125819-e735f78e58c6/go.mod h1:vHWauJsS6ryf7UDqq8hRXoYD5RsONxcFTxeZPOztEuI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/akamensky/argparse v1.4.0 h1:YGzvsTqCvbEZhL8zZu2AiA5nq805NZh75JNj4ajn1xc= github.com/akamensky/argparse v1.4.0/go.mod h1:S5kwC7IuDcEr5VeXtGPRVZ5o/FdhcMlQz4IZQuw64xA= @@ -252,6 +254,8 @@ golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -275,6 +279,8 @@ golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= @@ -290,6 +296,8 @@ golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -307,10 +315,14 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= +golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= +golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -319,6 +331,8 @@ golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/main.go b/main.go index 2e029ff..721a23b 100644 --- a/main.go +++ b/main.go @@ -102,16 +102,16 @@ func main() { workflowName = getContainerName(argoFilePath) - // if conf.GetConfig().KubeHost == "" { - // // Not in a k8s environment, get conf from parameters - // logger.Info().Msg("Executes outside of k8s") - // executeOutside(argoFilePath, builder.Workflow) - // } else { - // // Executed in a k8s environment - // logger.Info().Msg("Executes inside a k8s") - // // executeInside(exec.GetID(), "argo", argo_file_path, stepMax) // commenting to use conf.ExecutionID instead of exec.GetID() - // executeInside(conf.GetConfig().ExecutionID, exec.ExecutionsID, argoFilePath) - // } + if conf.GetConfig().KubeHost == "" { + // Not in a k8s environment, get conf from parameters + logger.Info().Msg("Executes outside of k8s") + executeOutside(argoFilePath, builder.Workflow) + } else { + // Executed in a k8s environment + logger.Info().Msg("Executes inside a k8s") + // executeInside(exec.GetID(), "argo", argo_file_path, stepMax) // commenting to use conf.ExecutionID instead of exec.GetID() + executeInside(conf.GetConfig().ExecutionID, exec.ExecutionsID, argoFilePath) + } } // So far we only log the output from diff --git a/workflow_builder/argo_builder.go b/workflow_builder/argo_builder.go index 3fbf04f..72862bf 100644 --- a/workflow_builder/argo_builder.go +++ b/workflow_builder/argo_builder.go @@ -55,7 +55,7 @@ func (b *Workflow) getDag() *Dag { } type Spec struct { - ServiceAccountName string `yaml:"serviceAccountName"` + ServiceAccountName string `yaml:"serviceAccountName,omitempty"` Entrypoint string `yaml:"entrypoint"` Arguments []Parameter `yaml:"arguments,omitempty"` Volumes []VolumeClaimTemplate `yaml:"volumeClaimTemplates,omitempty"` @@ -75,7 +75,7 @@ func (b *ArgoBuilder) CreateDAG(namespace string, write bool) ( int, []string, [ if b.Timeout > 0 { b.Workflow.Spec.Timeout = b.Timeout } - // b.Workflow.Spec.ServiceAccountName = "sa-"+namespace + b.Workflow.Spec.ServiceAccountName = "sa-" + namespace b.Workflow.Spec.Entrypoint = "dag" b.Workflow.ApiVersion = "argoproj.io/v1alpha1" b.Workflow.Kind = "Workflow" @@ -86,7 +86,7 @@ func (b *ArgoBuilder) CreateDAG(namespace string, write bool) ( int, []string, [ return len(b.Workflow.getDag().Tasks), firstItems, lastItems, nil } - + func (b *ArgoBuilder) createTemplates(namespace string) ([]string, []string, []VolumeMount) { volumes := []VolumeMount{} firstItems := []string{} @@ -260,7 +260,7 @@ func (b *ArgoBuilder) addS3annotations(art *Artifact, template *Template, rw gra // v1 : v0.2 + if doesn't exist edit/create the configMap with the response from API call if sel != nil { b.addAuthInformation(storage, namespace, art) - art.S3.Bucket = "oc-bucket" // DEFAULT : will need to update this to create an unique + art.S3.Bucket = namespace // DEFAULT : will need to update this to create an unique art.S3.EndPoint = sel.(*resources.StorageResourceInstance).Source } } @@ -490,21 +490,21 @@ func (b *ArgoBuilder) retrieveProcessingCompute(graphID string) *resources.Compu // Execute the last actions once the YAML file for the Argo Workflow is created func (b *ArgoBuilder) CompleteBuild(executionsId string) (string, error) { logger.Info().Msg(fmt.Sprint("DEV :: Completing build")) - // setter := AdmiraltySetter{Id: executionsId} - // // Setup admiralty for each node - // for _, peer := range b.RemotePeers { - // logger.Info().Msg(fmt.Sprint("DEV :: Launching Admiralty Setup for ", peer)) - // setter.InitializeAdmiralty(conf.GetConfig().PeerID,peer) - // } + setter := AdmiraltySetter{Id: executionsId} + // Setup admiralty for each node + for _, peer := range b.RemotePeers { + logger.Info().Msg(fmt.Sprint("DEV :: Launching Admiralty Setup for ", peer)) + setter.InitializeAdmiralty(conf.GetConfig().PeerID,peer) + } - // // Update the name of the admiralty node to use - // for _, template := range b.Workflow.Spec.Templates { - // if len(template.Metadata.Annotations) > 0 { - // if peerId, ok := template.Metadata.Annotations["multicluster.admiralty.io/clustername"]; ok { - // template.Metadata.Annotations["multicluster.admiralty.io/clustername"] = "target-" + peerId + "-" + conf.GetConfig().ExecutionID - // } - // } - // } + // Update the name of the admiralty node to use + for _, template := range b.Workflow.Spec.Templates { + if len(template.Metadata.Annotations) > 0 { + if peerId, ok := template.Metadata.Annotations["multicluster.admiralty.io/clustername"]; ok { + template.Metadata.Annotations["multicluster.admiralty.io/clustername"] = "target-" + oclib.GetConcatenatedName(peerId, executionsId) + } + } + } // Generate the YAML file random_name := fakelish.GenerateFakeWord(5, 8) + "-" + fakelish.GenerateFakeWord(5, 8) From d62a20ab39ff821b82dc2b62ca964f6a95684dec Mon Sep 17 00:00:00 2001 From: pb Date: Tue, 29 Jul 2025 11:43:45 +0200 Subject: [PATCH 9/9] Corrected the YAML annotation for access and secret KeySecret --- models/template.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/template.go b/models/template.go index 05dde12..99e5a51 100644 --- a/models/template.go +++ b/models/template.go @@ -71,8 +71,8 @@ type Key struct { Bucket string `yaml:"bucket"` EndPoint string `yaml:"endpoint"` Insecure bool `yaml:"insecure"` - AccessKeySecret *Secret `yaml: accessKeySecret` - SecretKeySecret *Secret `yaml: secretKeySecret` + AccessKeySecret *Secret `yaml:"accessKeySecret"` + SecretKeySecret *Secret `yaml:"secretKeySecret"` } type Artifact struct {