Merge branch 'main' of https://cloud.o-forge.io/core/oc-monitord into main
This commit is contained in:
commit
3e4ac40109
47
README.md
47
README.md
@ -15,3 +15,50 @@ imagePullPolicy: Never
|
|||||||
|
|
||||||
Not doing so will end up in the pod having a `ErrorImagePull`
|
Not doing so will end up in the pod having a `ErrorImagePull`
|
||||||
|
|
||||||
|
## Allow argo to create services
|
||||||
|
|
||||||
|
In order for monitord to expose **open cloud services** on the node, we need to give him permission to create **k8s services**.
|
||||||
|
|
||||||
|
For that we can update the RBAC configuration for a role already created by argo :
|
||||||
|
|
||||||
|
### Manually edit the rbac authorization
|
||||||
|
|
||||||
|
> kubectl edit roles.rbac.authorization.k8s.io -n argo argo-role
|
||||||
|
|
||||||
|
In rules add a new entry :
|
||||||
|
|
||||||
|
```
|
||||||
|
- apiGroups:
|
||||||
|
- ""
|
||||||
|
resources:
|
||||||
|
- services
|
||||||
|
verbs:
|
||||||
|
- get
|
||||||
|
- create
|
||||||
|
```
|
||||||
|
|
||||||
|
### Patch the rbac authorization with a one liner
|
||||||
|
|
||||||
|
> kubectl patch role argo-role -n argo --type='json' -p='[{"op": "add", "path": "/rules/-", "value": {"apiGroups": [""], "resources": ["services"], "verbs": ["get","create"]}}]'
|
||||||
|
|
||||||
|
### Check wether the modification is effective
|
||||||
|
|
||||||
|
> kubectl auth can-i create services --as=system:serviceaccount:argo:argo -n argo
|
||||||
|
|
||||||
|
This command **must return "yes"**
|
||||||
|
|
||||||
|
|
||||||
|
## TODO
|
||||||
|
|
||||||
|
- [ ] Logs the output of each pods :
|
||||||
|
- logsPods() function already exists
|
||||||
|
- need to implement the logic to create each pod's logger and start the monitoring routing
|
||||||
|
- [ ] Allow the front to known on which IP the service are reachable
|
||||||
|
- currently doing it by using `kubectl get nodes -o wide`
|
||||||
|
|
||||||
|
|
||||||
|
### Adding ingress handling to support reverse proxing
|
||||||
|
|
||||||
|
- Test wether ingress-nginx is running or not
|
||||||
|
- Do something if not found : stop running and send error log OR start installation
|
||||||
|
-
|
||||||
@ -3,10 +3,14 @@ package conf
|
|||||||
import "sync"
|
import "sync"
|
||||||
|
|
||||||
type Config struct {
|
type Config struct {
|
||||||
|
MongoURL string
|
||||||
|
Database string
|
||||||
LokiURL string
|
LokiURL string
|
||||||
|
NatsURL string
|
||||||
ExecutionID string
|
ExecutionID string
|
||||||
Timeout int
|
Timeout int
|
||||||
WorkflowID string
|
WorkflowID string
|
||||||
|
Logs string
|
||||||
}
|
}
|
||||||
|
|
||||||
var instance *Config
|
var instance *Config
|
||||||
|
|||||||
5
demo_nginx/DTF.html
Normal file
5
demo_nginx/DTF.html
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
<html>
|
||||||
|
<body>
|
||||||
|
<h1> This could be your DTF webpage</h1>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
5
demo_nginx/cockpit.html
Normal file
5
demo_nginx/cockpit.html
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
<html>
|
||||||
|
<body>
|
||||||
|
<h1> Welcome to your DTF Cockpit</h1>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@ -1,15 +1,13 @@
|
|||||||
{
|
{
|
||||||
"id": "9c7ffc7e-3e6e-4ea8-8eab-3a03258712ff",
|
"id": "9c7ffc7e-3e6e-4ea8-8eab-3a03258712ff",
|
||||||
"name": "test-services",
|
"name": "test-services",
|
||||||
"resourceset": {
|
"processings": [
|
||||||
"processings": [
|
"7c71a15b-bdbc-46d7-9dab-67e369804136",
|
||||||
"7c71a15b-bdbc-46d7-9dab-67e369804136",
|
"0d565c87-50ae-4a73-843d-f8b2d4047772"
|
||||||
"0d565c87-50ae-4a73-843d-f8b2d4047772"
|
],
|
||||||
],
|
"datacenters": [
|
||||||
"datacenters": [
|
"7b989e97-c3e7-49d2-a3a7-f959da4870b5"
|
||||||
"7b989e97-c3e7-49d2-a3a7-f959da4870b5"
|
],
|
||||||
]
|
|
||||||
},
|
|
||||||
"graph": {
|
"graph": {
|
||||||
"zoom": 1,
|
"zoom": 1,
|
||||||
"items": {
|
"items": {
|
||||||
@ -28,7 +26,7 @@
|
|||||||
},
|
},
|
||||||
"args": {
|
"args": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"value": "-SL https://cloud.o-forge.io/core/oc-monitord/raw/branch/services_demo/demo_nginx/cockpit.html -o /usr/share/ningx/cockpit.hmtl"
|
"value": "-SL https://cloud.o-forge.io/core/oc-monitord/raw/branch/services_demo/demo_nginx/cockpit.html -o /usr/share/nginx/cockpit.hmtl"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@ -61,7 +59,7 @@
|
|||||||
},
|
},
|
||||||
"args": {
|
"args": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"value": "-SL https://cloud.o-forge.io/core/oc-monitord/raw/branch/services_demo/demo_nginx/DTF.html -o /usr/share/ningx/DTF.hmtl"
|
"value": "-SL https://cloud.o-forge.io/core/oc-monitord/raw/branch/services_demo/demo_nginx/DTF.html -o /usr/share/nginx/DTF.hmtl"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@ -100,6 +98,15 @@
|
|||||||
},
|
},
|
||||||
"args": {
|
"args": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
|
},
|
||||||
|
"expose": {
|
||||||
|
"type": "dict",
|
||||||
|
"value": {
|
||||||
|
"80" :{
|
||||||
|
"reverse" : "",
|
||||||
|
"PAT" : "308080"
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@ -139,6 +146,15 @@
|
|||||||
},
|
},
|
||||||
"args": {
|
"args": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
|
},
|
||||||
|
"expose": {
|
||||||
|
"type": "dict",
|
||||||
|
"value": {
|
||||||
|
"80" :{
|
||||||
|
"reverse" : "",
|
||||||
|
"PAT" : "308081"
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@ -222,7 +238,7 @@
|
|||||||
"y": 0
|
"y": 0
|
||||||
},
|
},
|
||||||
"destination": {
|
"destination": {
|
||||||
"id": "0d565c87-50ae-4a73-843d-f8b2d4047772",
|
"id": "bf6916ff-b16f-44b3-818b-0bcd5bbaca00",
|
||||||
"x": 0,
|
"x": 0,
|
||||||
"y": 0
|
"y": 0
|
||||||
}
|
}
|
||||||
@ -277,12 +293,12 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"source": {
|
"source": {
|
||||||
"id": "6a7e8860-7c26-4b70-9b3a-1bd27adcdfe1",
|
"id": "d83ac451-4690-44d9-af09-48e7588b2db9",
|
||||||
"x": 0,
|
"x": 0,
|
||||||
"y": 0
|
"y": 0
|
||||||
},
|
},
|
||||||
"destination": {
|
"destination": {
|
||||||
"id": "d83ac451-4690-44d9-af09-48e7588b2db9",
|
"id": "6a7e8860-7c26-4b70-9b3a-1bd27adcdfe1",
|
||||||
"x": 0,
|
"x": 0,
|
||||||
"y": 0
|
"y": 0
|
||||||
}
|
}
|
||||||
|
|||||||
16
go.mod
16
go.mod
@ -3,7 +3,7 @@ module oc-monitord
|
|||||||
go 1.22.0
|
go 1.22.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
cloud.o-forge.io/core/oc-lib v0.0.0-20240830131445-af18dba5563c
|
cloud.o-forge.io/core/oc-lib v0.0.0-20240924075418-021b461b0a7d
|
||||||
github.com/akamensky/argparse v1.4.0
|
github.com/akamensky/argparse v1.4.0
|
||||||
github.com/goraz/onion v0.1.3
|
github.com/goraz/onion v0.1.3
|
||||||
github.com/nats-io/nats-server/v2 v2.10.18
|
github.com/nats-io/nats-server/v2 v2.10.18
|
||||||
@ -17,10 +17,10 @@ require (
|
|||||||
github.com/gabriel-vasile/mimetype v1.4.5 // indirect
|
github.com/gabriel-vasile/mimetype v1.4.5 // indirect
|
||||||
github.com/go-playground/locales v0.14.1 // indirect
|
github.com/go-playground/locales v0.14.1 // indirect
|
||||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||||
github.com/go-playground/validator/v10 v10.22.0 // indirect
|
github.com/go-playground/validator/v10 v10.22.1 // indirect
|
||||||
github.com/golang/snappy v0.0.4 // indirect
|
github.com/golang/snappy v0.0.4 // indirect
|
||||||
github.com/google/uuid v1.6.0 // indirect
|
github.com/google/uuid v1.6.0 // indirect
|
||||||
github.com/klauspost/compress v1.17.9 // indirect
|
github.com/klauspost/compress v1.17.10 // indirect
|
||||||
github.com/leodido/go-urn v1.4.0 // indirect
|
github.com/leodido/go-urn v1.4.0 // indirect
|
||||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||||
@ -38,10 +38,10 @@ require (
|
|||||||
github.com/xdg-go/scram v1.1.2 // indirect
|
github.com/xdg-go/scram v1.1.2 // indirect
|
||||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||||
go.mongodb.org/mongo-driver v1.16.1 // indirect
|
go.mongodb.org/mongo-driver v1.17.0 // indirect
|
||||||
golang.org/x/crypto v0.26.0 // indirect
|
golang.org/x/crypto v0.27.0 // indirect
|
||||||
golang.org/x/net v0.28.0 // indirect
|
golang.org/x/net v0.29.0 // indirect
|
||||||
golang.org/x/sync v0.8.0 // indirect
|
golang.org/x/sync v0.8.0 // indirect
|
||||||
golang.org/x/sys v0.24.0 // indirect
|
golang.org/x/sys v0.25.0 // indirect
|
||||||
golang.org/x/text v0.17.0 // indirect
|
golang.org/x/text v0.18.0 // indirect
|
||||||
)
|
)
|
||||||
|
|||||||
16
go.sum
16
go.sum
@ -44,6 +44,8 @@ cloud.o-forge.io/core/oc-lib v0.0.0-20240830071403-db78c70dc349 h1:bEIY1lCsA78/m
|
|||||||
cloud.o-forge.io/core/oc-lib v0.0.0-20240830071403-db78c70dc349/go.mod h1:FIJD0taWLJ5pjQLJ6sfE2KlTkvbmk5SMcyrxdjsaVz0=
|
cloud.o-forge.io/core/oc-lib v0.0.0-20240830071403-db78c70dc349/go.mod h1:FIJD0taWLJ5pjQLJ6sfE2KlTkvbmk5SMcyrxdjsaVz0=
|
||||||
cloud.o-forge.io/core/oc-lib v0.0.0-20240830131445-af18dba5563c h1:4ZoM9ONJiaeLHSi0s8gsCe4lHuRHXkfK+eDSnTCspa0=
|
cloud.o-forge.io/core/oc-lib v0.0.0-20240830131445-af18dba5563c h1:4ZoM9ONJiaeLHSi0s8gsCe4lHuRHXkfK+eDSnTCspa0=
|
||||||
cloud.o-forge.io/core/oc-lib v0.0.0-20240830131445-af18dba5563c/go.mod h1:FIJD0taWLJ5pjQLJ6sfE2KlTkvbmk5SMcyrxdjsaVz0=
|
cloud.o-forge.io/core/oc-lib v0.0.0-20240830131445-af18dba5563c/go.mod h1:FIJD0taWLJ5pjQLJ6sfE2KlTkvbmk5SMcyrxdjsaVz0=
|
||||||
|
cloud.o-forge.io/core/oc-lib v0.0.0-20240924075418-021b461b0a7d h1:f8cT/NunF+eoZLU5B9gmiT4ky99zPmnQBbj4tj23KuA=
|
||||||
|
cloud.o-forge.io/core/oc-lib v0.0.0-20240924075418-021b461b0a7d/go.mod h1:FIJD0taWLJ5pjQLJ6sfE2KlTkvbmk5SMcyrxdjsaVz0=
|
||||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||||
github.com/akamensky/argparse v1.4.0 h1:YGzvsTqCvbEZhL8zZu2AiA5nq805NZh75JNj4ajn1xc=
|
github.com/akamensky/argparse v1.4.0 h1:YGzvsTqCvbEZhL8zZu2AiA5nq805NZh75JNj4ajn1xc=
|
||||||
github.com/akamensky/argparse v1.4.0/go.mod h1:S5kwC7IuDcEr5VeXtGPRVZ5o/FdhcMlQz4IZQuw64xA=
|
github.com/akamensky/argparse v1.4.0/go.mod h1:S5kwC7IuDcEr5VeXtGPRVZ5o/FdhcMlQz4IZQuw64xA=
|
||||||
@ -69,6 +71,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
|
|||||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||||
github.com/go-playground/validator/v10 v10.22.0 h1:k6HsTZ0sTnROkhS//R0O+55JgM8C4Bx7ia+JlgcnOao=
|
github.com/go-playground/validator/v10 v10.22.0 h1:k6HsTZ0sTnROkhS//R0O+55JgM8C4Bx7ia+JlgcnOao=
|
||||||
github.com/go-playground/validator/v10 v10.22.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
|
github.com/go-playground/validator/v10 v10.22.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
|
||||||
|
github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA=
|
||||||
|
github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
|
||||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||||
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
|
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
|
||||||
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||||
@ -87,6 +91,8 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7
|
|||||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||||
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
|
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
|
||||||
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
|
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
|
||||||
|
github.com/klauspost/compress v1.17.10 h1:oXAz+Vh0PMUvJczoi+flxpnBEPxoER1IaAnU/NMPtT0=
|
||||||
|
github.com/klauspost/compress v1.17.10/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
|
||||||
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
|
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
|
||||||
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
|
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
|
||||||
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||||
@ -169,6 +175,8 @@ go.mongodb.org/mongo-driver v1.16.0 h1:tpRsfBJMROVHKpdGyc1BBEzzjDUWjItxbVSZ8Ls4B
|
|||||||
go.mongodb.org/mongo-driver v1.16.0/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw=
|
go.mongodb.org/mongo-driver v1.16.0/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw=
|
||||||
go.mongodb.org/mongo-driver v1.16.1 h1:rIVLL3q0IHM39dvE+z2ulZLp9ENZKThVfuvN/IiN4l8=
|
go.mongodb.org/mongo-driver v1.16.1 h1:rIVLL3q0IHM39dvE+z2ulZLp9ENZKThVfuvN/IiN4l8=
|
||||||
go.mongodb.org/mongo-driver v1.16.1/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw=
|
go.mongodb.org/mongo-driver v1.16.1/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw=
|
||||||
|
go.mongodb.org/mongo-driver v1.17.0 h1:Hp4q2MCjvY19ViwimTs00wHi7G4yzxh4/2+nTx8r40k=
|
||||||
|
go.mongodb.org/mongo-driver v1.17.0/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4=
|
||||||
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||||
@ -177,6 +185,8 @@ golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
|
|||||||
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
|
golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
|
||||||
golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
|
golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
|
||||||
golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
|
golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
|
||||||
|
golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
|
||||||
|
golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
|
||||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||||
@ -187,6 +197,8 @@ golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
|
|||||||
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
|
golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
|
||||||
golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
|
golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
|
||||||
golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
|
golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
|
||||||
|
golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
|
||||||
|
golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
|
||||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
|
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
|
||||||
@ -210,6 +222,8 @@ golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
|
|||||||
golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
|
golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
|
||||||
golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
|
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
|
||||||
|
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
@ -220,6 +234,8 @@ golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
|
|||||||
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
|
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
|
||||||
golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
|
golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
|
||||||
golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
||||||
|
golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
|
||||||
|
golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
||||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||||
|
|||||||
31
main.go
31
main.go
@ -21,7 +21,6 @@ import (
|
|||||||
"cloud.o-forge.io/core/oc-lib/logs"
|
"cloud.o-forge.io/core/oc-lib/logs"
|
||||||
"cloud.o-forge.io/core/oc-lib/models/utils"
|
"cloud.o-forge.io/core/oc-lib/models/utils"
|
||||||
"cloud.o-forge.io/core/oc-lib/models/workflow_execution"
|
"cloud.o-forge.io/core/oc-lib/models/workflow_execution"
|
||||||
"cloud.o-forge.io/core/oc-lib/tools"
|
|
||||||
|
|
||||||
"github.com/akamensky/argparse"
|
"github.com/akamensky/argparse"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
@ -64,13 +63,19 @@ func main() {
|
|||||||
loadConfig(true, nil)
|
loadConfig(true, nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
logs.SetAppName("oc-monitord")
|
logger = logs.CreateLogger("oc-monitord")
|
||||||
logger = logs.CreateLogger("oc-monitord", conf.GetConfig().LokiURL)
|
|
||||||
|
|
||||||
logger.Debug().Msg("Loki URL : " + conf.GetConfig().LokiURL)
|
logger.Debug().Msg("Loki URL : " + conf.GetConfig().LokiURL)
|
||||||
logger.Debug().Msg("Workflow executed : " + conf.GetConfig().ExecutionID)
|
logger.Debug().Msg("Workflow executed : " + conf.GetConfig().ExecutionID)
|
||||||
|
|
||||||
oclib.Init("oc-monitord", "", "")
|
oclib.SetConfig(
|
||||||
|
conf.GetConfig().MongoURL,
|
||||||
|
conf.GetConfig().Database,
|
||||||
|
conf.GetConfig().NatsURL,
|
||||||
|
conf.GetConfig().LokiURL,
|
||||||
|
conf.GetConfig().Logs,
|
||||||
|
)
|
||||||
|
oclib.Init("oc-monitord")
|
||||||
|
|
||||||
wf_id := getWorkflowId(conf.GetConfig().ExecutionID)
|
wf_id := getWorkflowId(conf.GetConfig().ExecutionID)
|
||||||
conf.GetConfig().WorkflowID = wf_id
|
conf.GetConfig().WorkflowID = wf_id
|
||||||
@ -121,6 +126,8 @@ func getWorkflowId(exec_id string) string {
|
|||||||
return wf_exec.WorkflowID
|
return wf_exec.WorkflowID
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// So far we only log the output from
|
||||||
|
|
||||||
func executeWorkflow(argo_file_path string) {
|
func executeWorkflow(argo_file_path string) {
|
||||||
// var stdout, stderr, stdout_logs, stderr_logs io.ReadCloser
|
// var stdout, stderr, stdout_logs, stderr_logs io.ReadCloser
|
||||||
var stdout, stderr io.ReadCloser
|
var stdout, stderr io.ReadCloser
|
||||||
@ -187,7 +194,7 @@ func logWorkflow(pipe io.ReadCloser, wg *sync.WaitGroup) {
|
|||||||
|
|
||||||
// Debug, no logs sent
|
// Debug, no logs sent
|
||||||
func logPods(pipe io.ReadCloser, name string) {
|
func logPods(pipe io.ReadCloser, name string) {
|
||||||
pods_logger = wf_logger.With().Str("sortie name", name).Logger()
|
pods_logger = wf_logger.With().Str("pod_name", name).Logger()
|
||||||
scanner := bufio.NewScanner(pipe)
|
scanner := bufio.NewScanner(pipe)
|
||||||
for scanner.Scan() {
|
for scanner.Scan() {
|
||||||
log := scanner.Text()
|
log := scanner.Text()
|
||||||
@ -202,7 +209,7 @@ func loadConfig(is_k8s bool, parser *argparse.Parser) {
|
|||||||
|
|
||||||
o = initOnion(o)
|
o = initOnion(o)
|
||||||
// These variables can only be retrieved in the onion
|
// These variables can only be retrieved in the onion
|
||||||
// Variables that don't depend on the environmen (from conf file), can be loaded after
|
// Variables that don't depend on the environment (from conf file), can be loaded after
|
||||||
// We can't use underscore in the env variable names because it's the delimitor with OCMONITOR too
|
// We can't use underscore in the env variable names because it's the delimitor with OCMONITOR too
|
||||||
setConf(is_k8s, o, parser)
|
setConf(is_k8s, o, parser)
|
||||||
|
|
||||||
@ -223,7 +230,9 @@ func setConf(is_k8s bool, o *onion.Onion, parser *argparse.Parser) {
|
|||||||
conf.GetConfig().ExecutionID = o.GetString("workflow")
|
conf.GetConfig().ExecutionID = o.GetString("workflow")
|
||||||
mongo := o.GetStringDefault("mongourl", "mongodb://127.0.0.1:27017")
|
mongo := o.GetStringDefault("mongourl", "mongodb://127.0.0.1:27017")
|
||||||
db := o.GetStringDefault("database", "DC_myDC")
|
db := o.GetStringDefault("database", "DC_myDC")
|
||||||
tools.SetConfig(mongo, db, "")
|
|
||||||
|
conf.GetConfig().MongoURL = mongo
|
||||||
|
conf.GetConfig().Database = db
|
||||||
} else {
|
} else {
|
||||||
url := parser.String("u", "url", &argparse.Options{Required: true, Default: "http://127.0.0.1:3100", Help: "Url to the Loki database logs will be sent to"})
|
url := parser.String("u", "url", &argparse.Options{Required: true, Default: "http://127.0.0.1:3100", Help: "Url to the Loki database logs will be sent to"})
|
||||||
execution := parser.String("e", "execution", &argparse.Options{Required: true, Help: "Execution ID of the workflow to request from oc-catalog API"})
|
execution := parser.String("e", "execution", &argparse.Options{Required: true, Help: "Execution ID of the workflow to request from oc-catalog API"})
|
||||||
@ -235,16 +244,20 @@ func setConf(is_k8s bool, o *onion.Onion, parser *argparse.Parser) {
|
|||||||
fmt.Println(parser.Usage(err))
|
fmt.Println(parser.Usage(err))
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
conf.GetConfig().Logs = "debug"
|
||||||
|
|
||||||
conf.GetConfig().LokiURL = *url
|
conf.GetConfig().LokiURL = *url
|
||||||
|
conf.GetConfig().MongoURL = *mongo
|
||||||
|
conf.GetConfig().Database = *db
|
||||||
conf.GetConfig().Timeout = *timeout
|
conf.GetConfig().Timeout = *timeout
|
||||||
conf.GetConfig().ExecutionID = *execution
|
conf.GetConfig().ExecutionID = *execution
|
||||||
tools.SetConfig(*mongo, *db, "")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func initOnion(o *onion.Onion) *onion.Onion {
|
func initOnion(o *onion.Onion) *onion.Onion {
|
||||||
logger = logs.CreateLogger("oc-monitord", "")
|
logger = logs.CreateLogger("oc-monitord")
|
||||||
configFile := ""
|
configFile := ""
|
||||||
|
|
||||||
l3 := onion.NewEnvLayerPrefix("_", "OCMONITORD")
|
l3 := onion.NewEnvLayerPrefix("_", "OCMONITORD")
|
||||||
|
|||||||
40
models/services.go
Normal file
40
models/services.go
Normal file
@ -0,0 +1,40 @@
|
|||||||
|
package models
|
||||||
|
|
||||||
|
|
||||||
|
type ServiceResource struct {
|
||||||
|
Action string `yaml:"action,omitempty"`
|
||||||
|
SuccessCondition string `yaml:"successCondition,omitempty"`
|
||||||
|
FailureCondition string `yaml:"failureCondition,omitempty"`
|
||||||
|
SetOwnerReference bool `yaml:"setOwnerReference,omitempty"`
|
||||||
|
Manifest string `yaml:"manifest,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Service struct {
|
||||||
|
APIVersion string `yaml:"apiVersion"`
|
||||||
|
Kind string `yaml:"kind"`
|
||||||
|
Metadata Metadata `yaml:"metadata"`
|
||||||
|
Spec ServiceSpec `yaml:"spec"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Metadata struct {
|
||||||
|
Name string `yaml:"name"`
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// ServiceSpec is the specification of the Kubernetes Service
|
||||||
|
type ServiceSpec struct {
|
||||||
|
Selector map[string]string `yaml:"selector,omitempty"`
|
||||||
|
Ports []ServicePort `yaml:"ports"`
|
||||||
|
ClusterIP string `yaml:"clusterIP,omitempty"`
|
||||||
|
Type string `yaml:"type,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ServicePort defines a port for a Kubernetes Service
|
||||||
|
type ServicePort struct {
|
||||||
|
Name string `yaml:"name"` // Even if empty need to be in the yaml
|
||||||
|
|
||||||
|
Protocol string `yaml:"protocol,omitempty"`
|
||||||
|
Port int64 `yaml:"port"`
|
||||||
|
TargetPort int64 `yaml:"targetPort,omitempty"`
|
||||||
|
NodePort int64 `yaml:"nodePort,omitempty"`
|
||||||
|
}
|
||||||
@ -30,6 +30,10 @@ type Dag struct {
|
|||||||
Tasks []Task `yaml:"tasks,omitempty"`
|
Tasks []Task `yaml:"tasks,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type TemplateMetadata struct {
|
||||||
|
Labels map[string]string `yaml:"labels,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
type Template struct {
|
type Template struct {
|
||||||
Name string `yaml:"name"`
|
Name string `yaml:"name"`
|
||||||
Inputs struct {
|
Inputs struct {
|
||||||
@ -37,4 +41,6 @@ type Template struct {
|
|||||||
} `yaml:"inputs,omitempty"`
|
} `yaml:"inputs,omitempty"`
|
||||||
Container Container `yaml:"container,omitempty"`
|
Container Container `yaml:"container,omitempty"`
|
||||||
Dag Dag `yaml:"dag,omitempty"`
|
Dag Dag `yaml:"dag,omitempty"`
|
||||||
|
Metadata TemplateMetadata `yaml:"metadata,omitempty"`
|
||||||
|
Resource ServiceResource `yaml:"resource,omitempty"`
|
||||||
}
|
}
|
||||||
|
|||||||
8
models/translate_ports.go
Normal file
8
models/translate_ports.go
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
package models
|
||||||
|
|
||||||
|
type PortTranslation map[string]PortConfig
|
||||||
|
|
||||||
|
type PortConfig struct {
|
||||||
|
Reverse string `json:"reverse,omitempty"`
|
||||||
|
PAT string `json:"PAT,omitempty"`
|
||||||
|
}
|
||||||
141
test-logs-workflow.json
Normal file
141
test-logs-workflow.json
Normal file
@ -0,0 +1,141 @@
|
|||||||
|
{
|
||||||
|
"name": "test-log",
|
||||||
|
"processings": [
|
||||||
|
"0d565c87-50ae-4a73-843d-f8b2d4047772",
|
||||||
|
"2ce0323f-a85d-4b8b-a783-5280f48d634a"
|
||||||
|
],
|
||||||
|
"datacenters": [
|
||||||
|
"7b989e97-c3e7-49d2-a3a7-f959da4870b5"
|
||||||
|
],
|
||||||
|
"graph": {
|
||||||
|
"items": {
|
||||||
|
"aa8d2265-9fe2-42c7-ba1f-46ea0da8e633": {
|
||||||
|
"id": "aa8d2265-9fe2-42c7-ba1f-46ea0da8e633",
|
||||||
|
"width": 0,
|
||||||
|
"height": 0,
|
||||||
|
"processing": {
|
||||||
|
"id": "0d565c87-50ae-4a73-843d-f8b2d4047772",
|
||||||
|
"resource_model": {
|
||||||
|
"resource_type": "processing",
|
||||||
|
"model": {
|
||||||
|
"command": {
|
||||||
|
"type": "string",
|
||||||
|
"value": "curlimages/curl:7.88.1"
|
||||||
|
},
|
||||||
|
"args": {
|
||||||
|
"type": "string",
|
||||||
|
"value": "-SL https://toulousefc.com -o /mnt/vol/tfc.hmtl"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"name": "CURL",
|
||||||
|
"short_description": "Transfer or retrieve information from or to a server ",
|
||||||
|
"description": "curl is a tool for transferring data from or to a server. It supports these protocols: DICT, FILE, FTP, FTPS, GOPHER, GOPHERS, HTTP, HTTPS, IMAP, IMAPS, LDAP, LDAPS, MQTT, POP3, POP3S, RTMP, RTMPS, RTSP, SCP, SFTP, SMB, SMBS, SMTP, SMTPS, TELNET, TFTP, WS and WSS.",
|
||||||
|
"logo": "https://cloud.o-forge.io/core/deperecated-oc-catalog/src/branch/main/scripts/local_imgs/curl-logo.png",
|
||||||
|
"owner": "IRT",
|
||||||
|
"source_url": "http://www.google.com",
|
||||||
|
"license": "GPLv2",
|
||||||
|
"ram": {},
|
||||||
|
"storage": 300,
|
||||||
|
"parallel": true,
|
||||||
|
"scaling_model": 2,
|
||||||
|
"disk_io": "30 MB/s"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"b6f790ae-5db2-4458-8375-59450f12f65b": {
|
||||||
|
"id": "b6f790ae-5db2-4458-8375-59450f12f65b",
|
||||||
|
"width": 0,
|
||||||
|
"height": 0,
|
||||||
|
"processing": {
|
||||||
|
"id": "2ce0323f-a85d-4b8b-a783-5280f48d634a",
|
||||||
|
"resource_model": {
|
||||||
|
"resource_type": "processing",
|
||||||
|
"model": {
|
||||||
|
"command": {
|
||||||
|
"type": "string",
|
||||||
|
"value": "alpine:3.7 sh -c"
|
||||||
|
},
|
||||||
|
"args": {
|
||||||
|
"type": "string",
|
||||||
|
"value": "grep '<meta name=\"description\"' /mnt/vol/tfc.html | awk -F 'content=\"' '{print $2}' | awk -F '\"' '{print $1}'"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"name": "alpine",
|
||||||
|
"short_description": "A minimal Docker image ",
|
||||||
|
"description": "Alpine Linux is a Linux distribution built around musl libc and BusyBox. The image is only 5 MB in size and has access to a package repository that is much more complete than other BusyBox based images. This makes Alpine Linux a great image base for utilities and even production applications",
|
||||||
|
"logo": "https://cloud.o-forge.io/core/deperecated-oc-catalog/src/branch/main/scripts/local_imgs/alpine-logo.png",
|
||||||
|
"owner": "IRT",
|
||||||
|
"source_url": "http://www.google.com",
|
||||||
|
"license": "GPLv2",
|
||||||
|
"ram": {},
|
||||||
|
"storage": 300,
|
||||||
|
"parallel": true,
|
||||||
|
"scaling_model": 2,
|
||||||
|
"disk_io": "30 MB/s"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"6a7e8860-7c26-4b70-9b3a-1bd27adcdfe1": {
|
||||||
|
"id": "6a7e8860-7c26-4b70-9b3a-1bd27adcdfe1",
|
||||||
|
"width": 0,
|
||||||
|
"height": 0,
|
||||||
|
"datacenter" : {
|
||||||
|
"id": "7b989e97-c3e7-49d2-a3a7-f959da4870b5",
|
||||||
|
"name": "Mundi datacenter",
|
||||||
|
"short_description": "Mundi Opencloud Instance",
|
||||||
|
"description": "A very long description of what this data is",
|
||||||
|
"logo": "https://cloud.o-forge.io/core/deperecated-oc-catalog/src/branch/main/scripts/local_imgs/Mundi datacenter.png",
|
||||||
|
"owner": "IRT",
|
||||||
|
"source_url": "http://www.google.com",
|
||||||
|
"resource_model": {
|
||||||
|
"id": "c3983010-1990-4ac0-8533-5389867e4424",
|
||||||
|
"resource_type": "datacenter_resource"
|
||||||
|
},
|
||||||
|
"cpus": [
|
||||||
|
{
|
||||||
|
"cores": 8,
|
||||||
|
"architecture": "x86"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"ram": {
|
||||||
|
"size": 16384
|
||||||
|
},
|
||||||
|
"gpus": [
|
||||||
|
{
|
||||||
|
"cuda_cores": 10496,
|
||||||
|
"model": "RTX 3090 FE",
|
||||||
|
"memory": 24000,
|
||||||
|
"tensor_cores": 328
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"links": [
|
||||||
|
{
|
||||||
|
"source": {
|
||||||
|
"id": "aa8d2265-9fe2-42c7-ba1f-46ea0da8e633"
|
||||||
|
},
|
||||||
|
"destination": {
|
||||||
|
"id": "b6f790ae-5db2-4458-8375-59450f12f65b"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source": {
|
||||||
|
"id": "aa8d2265-9fe2-42c7-ba1f-46ea0da8e633"
|
||||||
|
},
|
||||||
|
"destination": {
|
||||||
|
"id": "6a7e8860-7c26-4b70-9b3a-1bd27adcdfe1"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source": {
|
||||||
|
"id": "6a7e8860-7c26-4b70-9b3a-1bd27adcdfe1"
|
||||||
|
},
|
||||||
|
"destination": {
|
||||||
|
"id": "b6f790ae-5db2-4458-8375-59450f12f65b"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -16,13 +16,17 @@ import (
|
|||||||
"cloud.o-forge.io/core/oc-lib/models/resources/workflow/graph"
|
"cloud.o-forge.io/core/oc-lib/models/resources/workflow/graph"
|
||||||
w "cloud.o-forge.io/core/oc-lib/models/workflow"
|
w "cloud.o-forge.io/core/oc-lib/models/workflow"
|
||||||
"github.com/nwtgck/go-fakelish"
|
"github.com/nwtgck/go-fakelish"
|
||||||
|
"github.com/rs/zerolog"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var logger zerolog.Logger
|
||||||
|
|
||||||
type ArgoBuilder struct {
|
type ArgoBuilder struct {
|
||||||
OriginWorkflow w.Workflow
|
OriginWorkflow w.Workflow
|
||||||
Workflow Workflow
|
Workflow Workflow
|
||||||
Timeout int
|
Services *Service
|
||||||
|
Timeout int
|
||||||
}
|
}
|
||||||
|
|
||||||
type Workflow struct {
|
type Workflow struct {
|
||||||
@ -45,9 +49,9 @@ type Spec struct {
|
|||||||
func (b *ArgoBuilder) CreateDAG() (string, error) {
|
func (b *ArgoBuilder) CreateDAG() (string, error) {
|
||||||
|
|
||||||
// handle services by checking if there is only one processing with hostname and port
|
// handle services by checking if there is only one processing with hostname and port
|
||||||
if (b.isService()){
|
|
||||||
b.createNginxVolumes()
|
b.createNginxVolumes()
|
||||||
}
|
|
||||||
|
|
||||||
b.createTemplates()
|
b.createTemplates()
|
||||||
b.createDAGstep()
|
b.createDAGstep()
|
||||||
@ -62,7 +66,7 @@ func (b *ArgoBuilder) CreateDAG() (string, error) {
|
|||||||
b.Workflow.Kind = "Workflow"
|
b.Workflow.Kind = "Workflow"
|
||||||
random_name := generateWfName()
|
random_name := generateWfName()
|
||||||
b.Workflow.Metadata.Name = "oc-monitor-" + random_name
|
b.Workflow.Metadata.Name = "oc-monitor-" + random_name
|
||||||
logger := oclib.GetLogger()
|
logger = oclib.GetLogger()
|
||||||
yamlified, err := yaml.Marshal(b.Workflow)
|
yamlified, err := yaml.Marshal(b.Workflow)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Error().Msg("Could not transform object to yaml file")
|
logger.Error().Msg("Could not transform object to yaml file")
|
||||||
@ -111,8 +115,22 @@ func (b *ArgoBuilder) createTemplates() {
|
|||||||
new_temp.Inputs.Parameters = inputs_container
|
new_temp.Inputs.Parameters = inputs_container
|
||||||
new_temp.Container.VolumeMounts = append(new_temp.Container.VolumeMounts, VolumeMount{Name: "workdir", MountPath: "/mnt/vol"}) // TODO : replace this with a search of the storage / data source name
|
new_temp.Container.VolumeMounts = append(new_temp.Container.VolumeMounts, VolumeMount{Name: "workdir", MountPath: "/mnt/vol"}) // TODO : replace this with a search of the storage / data source name
|
||||||
new_temp.Container.VolumeMounts = append(new_temp.Container.VolumeMounts, VolumeMount{Name: "nginx-demo", MountPath: "/usr/share/nginx"}) // Used for processing services' demo with nginx
|
new_temp.Container.VolumeMounts = append(new_temp.Container.VolumeMounts, VolumeMount{Name: "nginx-demo", MountPath: "/usr/share/nginx"}) // Used for processing services' demo with nginx
|
||||||
b.Workflow.Spec.Templates = append(b.Workflow.Spec.Templates, new_temp)
|
|
||||||
|
|
||||||
|
if (b.isService(comp.ID)){
|
||||||
|
serv := b.CreateService(comp)
|
||||||
|
b.createService(serv, argo_name, comp.ID)
|
||||||
|
new_temp.Metadata.Labels = make(map[string]string)
|
||||||
|
new_temp.Metadata.Labels["app"] = "oc-service" // Construct the template for the k8s service and add a link in graph between k8s service and processing
|
||||||
|
// if err != nil {
|
||||||
|
// // TODO
|
||||||
|
// }
|
||||||
|
}
|
||||||
|
|
||||||
|
b.Workflow.Spec.Templates = append(b.Workflow.Spec.Templates, new_temp)
|
||||||
|
}
|
||||||
|
|
||||||
|
if b.Services != nil {
|
||||||
|
b.addServiceToArgo()
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
@ -134,6 +152,11 @@ func (b *ArgoBuilder) createDAGstep() {
|
|||||||
step.Dependencies = b.getDependency(comp.ID) // Error : we use the component ID instead of the GraphItem ID -> store objects
|
step.Dependencies = b.getDependency(comp.ID) // Error : we use the component ID instead of the GraphItem ID -> store objects
|
||||||
new_dag.Tasks = append(new_dag.Tasks, step)
|
new_dag.Tasks = append(new_dag.Tasks, step)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if b.Services != nil {
|
||||||
|
new_dag.Tasks = append(new_dag.Tasks, Task{Name:"workflow-service-pod", Template: "workflow-service-pod"})
|
||||||
|
}
|
||||||
|
|
||||||
b.Workflow.Spec.Templates = append(b.Workflow.Spec.Templates, Template{Name: "dag", Dag: new_dag})
|
b.Workflow.Spec.Templates = append(b.Workflow.Spec.Templates, Template{Name: "dag", Dag: new_dag})
|
||||||
|
|
||||||
}
|
}
|
||||||
@ -160,7 +183,7 @@ func (b *ArgoBuilder) createNginxVolumes() {
|
|||||||
|
|
||||||
func (b *ArgoBuilder) getDependency(current_computing_id string) (dependencies []string) {
|
func (b *ArgoBuilder) getDependency(current_computing_id string) (dependencies []string) {
|
||||||
for _, link := range b.OriginWorkflow.Graph.Links {
|
for _, link := range b.OriginWorkflow.Graph.Links {
|
||||||
if !b.IsProcessing(link.Source.ID) {
|
if b.OriginWorkflow.Graph.Items[link.Source.ID].Processing == nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
source := b.OriginWorkflow.Graph.Items[link.Source.ID].Processing
|
source := b.OriginWorkflow.Graph.Items[link.Source.ID].Processing
|
||||||
@ -272,8 +295,9 @@ func (b *ArgoBuilder) getProcessings() (list_computings []graph.GraphItem) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *ArgoBuilder) IsProcessing(id string) bool {
|
// Pass a GraphItem's UUID and not the ID
|
||||||
return slices.Contains(b.OriginWorkflow.Processings, id)
|
func (b *ArgoBuilder) IsProcessing(component_uuid string) bool {
|
||||||
|
return slices.Contains(b.OriginWorkflow.Processings, component_uuid)
|
||||||
}
|
}
|
||||||
|
|
||||||
func getStringValue(comp resource_model.AbstractResource, key string) string {
|
func getStringValue(comp resource_model.AbstractResource, key string) string {
|
||||||
@ -283,19 +307,26 @@ func getStringValue(comp resource_model.AbstractResource, key string) string {
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *ArgoBuilder) isService() bool{
|
func (b *ArgoBuilder) isService(id string) bool{
|
||||||
// for dev purpose do not commit to main
|
|
||||||
if os.Getenv("test_service") != ""{
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
comp_list := b.getProcessings()
|
comp := b.OriginWorkflow.Graph.Items[id]
|
||||||
|
|
||||||
if len(comp_list) != 1 {
|
if comp.Processing == nil {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
comp := comp_list[0]
|
_, is_exposed := comp.Processing.ResourceModel.Model["expose"]
|
||||||
|
return is_exposed
|
||||||
return comp.Data.ResourceModel.Model["port"].Value != "" && comp.Data.ResourceModel.Model["hostname"].Value != ""
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
func (b *ArgoBuilder) addLabel(name string, id string) {
|
||||||
|
argo_name := getArgoName(name,id)
|
||||||
|
for _, template := range b.Workflow.Spec.Templates{
|
||||||
|
if template.Name == argo_name{
|
||||||
|
template.Metadata.Labels["app"] = "service-workflow"
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|||||||
148
workflow_builder/argo_services.go
Normal file
148
workflow_builder/argo_services.go
Normal file
@ -0,0 +1,148 @@
|
|||||||
|
package workflow_builder
|
||||||
|
|
||||||
|
import (
|
||||||
|
"oc-monitord/models"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"cloud.o-forge.io/core/oc-lib/models/resource_model"
|
||||||
|
"cloud.o-forge.io/core/oc-lib/models/resources/workflow/graph"
|
||||||
|
"go.mongodb.org/mongo-driver/bson"
|
||||||
|
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||||
|
"gopkg.in/yaml.v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TODO : refactor this method or the deserialization process in oc-lib to get rid of the mongo code
|
||||||
|
func getExposeContract(expose resource_model.Model) map[string]map[string]string {
|
||||||
|
contract := make(map[string]map[string]string,0)
|
||||||
|
|
||||||
|
mapped_info := bson.M{}
|
||||||
|
// var contract PortTranslation
|
||||||
|
_ , byt, _ := bson.MarshalValue(expose.Value)
|
||||||
|
|
||||||
|
bson.Unmarshal(byt,&mapped_info)
|
||||||
|
|
||||||
|
for _,v := range mapped_info {
|
||||||
|
port := v.(primitive.M)["Key"].(string)
|
||||||
|
// exposed_port := map[string]interface{}{data["Key"] : ""}
|
||||||
|
port_translation := v.(primitive.M)["Value"]
|
||||||
|
contract[port] = map[string]string{}
|
||||||
|
for _,v2 := range port_translation.(primitive.A) {
|
||||||
|
if v2.(primitive.M)["Key"] == "reverse" {
|
||||||
|
contract[port]["reverse"] = v2.(primitive.M)["Value"].(string)
|
||||||
|
}
|
||||||
|
if v2.(primitive.M)["Key"] == "PAT" {
|
||||||
|
contract[port]["PAT"] = v2.(primitive.M)["Value"].(string)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return contract
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
func (b *ArgoBuilder) CreateService(processing graph.GraphItem) models.Service{
|
||||||
|
|
||||||
|
// model {
|
||||||
|
// Type : "dict",
|
||||||
|
// Value : {
|
||||||
|
// "80" : {
|
||||||
|
// "reverse" : "",
|
||||||
|
// "PAT" : "34000"
|
||||||
|
// },
|
||||||
|
// "344" : {
|
||||||
|
// "reverse" : "",
|
||||||
|
// "PAT" : "34400"
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
|
||||||
|
new_service := models.Service{APIVersion: "v1",
|
||||||
|
Kind: "Service",
|
||||||
|
Metadata: models.Metadata{
|
||||||
|
Name: "workflow-service" ,
|
||||||
|
},
|
||||||
|
Spec: models.ServiceSpec{
|
||||||
|
Selector: map[string]string{"app": "oc-service"},
|
||||||
|
Ports: []models.ServicePort{
|
||||||
|
},
|
||||||
|
Type: "NodePort",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
completeServicePorts(&new_service, processing)
|
||||||
|
yamlified, _ := yaml.Marshal(new_service)
|
||||||
|
x := string(yamlified)
|
||||||
|
_ = x
|
||||||
|
return new_service
|
||||||
|
}
|
||||||
|
|
||||||
|
func completeServicePorts(service *models.Service, processing graph.GraphItem) {
|
||||||
|
|
||||||
|
contract := getExposeContract(processing.Processing.ResourceModel.Model["expose"])
|
||||||
|
|
||||||
|
|
||||||
|
for str_port,translation_dict := range contract{
|
||||||
|
|
||||||
|
port, err := strconv.ParseInt(str_port, 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error().Msg("Could not convert " + str_port + "to an int")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if _, ok := translation_dict["PAT"]; ok{
|
||||||
|
port_translation, err := strconv.ParseInt(translation_dict["PAT"], 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error().Msg("Could not convert " + translation_dict["PAT"] + "to an int")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
new_port_translation := models.ServicePort{
|
||||||
|
Name: strings.ToLower(processing.Processing.Name) + processing.ID,
|
||||||
|
Port: port_translation-30000,
|
||||||
|
TargetPort: port,
|
||||||
|
NodePort: port_translation,
|
||||||
|
Protocol: "TCP",
|
||||||
|
}
|
||||||
|
service.Spec.Ports = append(service.Spec.Ports, new_port_translation)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *ArgoBuilder) createService(service models.Service, processing_name string, processing_id string) {
|
||||||
|
if b.Services != nil{
|
||||||
|
b.Services.Spec.Ports = append(b.Services.Spec.Ports, service.Spec.Ports...)
|
||||||
|
}else {
|
||||||
|
b.Services = &service
|
||||||
|
}
|
||||||
|
|
||||||
|
b.addLabel(processing_name,processing_id)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *ArgoBuilder) addServiceToArgo() error {
|
||||||
|
service_manifest, err := yaml.Marshal(b.Services)
|
||||||
|
if err != nil {
|
||||||
|
logger.Error().Msg("Could not marshal service manifest")
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
service_template := models.Template{Name: "workflow-service-pod",
|
||||||
|
Resource: models.ServiceResource{
|
||||||
|
Action: "create",
|
||||||
|
SuccessCondition: "status.succeeded > 0",
|
||||||
|
FailureCondition: "status.failed > 3",
|
||||||
|
SetOwnerReference: true,
|
||||||
|
Manifest: string(service_manifest),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
b.Workflow.Spec.Templates = append(b.Workflow.Spec.Templates, service_template)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user