core/oc-monitord
6
0
Fork 0
Code Issues 3 Pull Requests Packages Projects Releases Wiki Activity

Monitor With Data Storage + Datas

Browse Source
This commit is contained in:
mr
2024-10-11 13:44:16 +02:00
parent f388351476
commit 310395f869
10 changed files with 532 additions and 491 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
models/argo_logs.go
View File

@@ -1,7 +1,12 @@
package models
import (
"fmt"
"strconv"
"strings"
"time"
"github.com/acarl005/stripansi"
)
type ArgoWatch struct {
@@ -13,6 +18,7 @@ type ArgoWatch struct {
Started string
Duration string
Progress string
Logs []string
}
type Conditions struct {
@@ -20,53 +26,103 @@ type Conditions struct {
Completed bool
}
func (a *ArgoWatch) Equals(arg ArgoWatch) bool {
func (a *ArgoWatch) Equals(arg *ArgoWatch) bool {
if arg == nil {
return false
}
return a.Status == arg.Status && a.Progress == arg.Progress && a.Conditions.PodRunning == arg.Conditions.PodRunning && a.Conditions.Completed == arg.Conditions.Completed
}
// Take the slice of string that make up one round of stderr outputs from the --watch option in argo submit
func NewArgoLogs(inputs []string) *ArgoWatch {
var workflow ArgoWatch
func NewArgoLogs(name string, namespace string, stepMax int) *ArgoLogs {
return &ArgoLogs{
Name: "oc-monitor-" + name,
Namespace: namespace,
CreatedDate: time.Now().Format("2006-01-02 15:04:05"),
StepCount: 0,
StepMax: stepMax,
stop: false,
}
}
type ArgoLogs struct {
Name string
Namespace string
CreatedDate string
StepCount int
StepMax int
stop bool
Started time.Time
}
func (a *ArgoLogs) StartStepRecording() {
a.StepCount += 1
a.Started = time.Now()
}
func (a *ArgoLogs) StopStepRecording(inputs []string) *ArgoWatch {
fn := strings.Split(a.Name, "_")
logs := []string{}
err := false
end := ""
for _, input := range inputs {
line := strings.TrimSpace(input)
if line == "" {
if line == "" || !strings.Contains(line, fn[0]) || !strings.Contains(line, ":") {
continue
}
switch {
case strings.HasPrefix(line, "Name:"):
workflow.Name = parseValue(line)
case strings.HasPrefix(line, "Namespace:"):
workflow.Namespace = parseValue(line)
case strings.HasPrefix(line, "Status:"):
workflow.Status = parseValue(line)
case strings.HasPrefix(line, "PodRunning"):
workflow.PodRunning = parseBoolValue(line)
case strings.HasPrefix(line, "Completed"):
workflow.Completed = parseBoolValue(line)
case strings.HasPrefix(line, "Created:"):
workflow.Created = parseValue(line)
case strings.HasPrefix(line, "Started:"):
workflow.Started = parseValue(line)
case strings.HasPrefix(line, "Duration:"):
workflow.Duration = parseValue(line)
case strings.HasPrefix(line, "Progress:"):
workflow.Progress = parseValue(line)
step := strings.Split(line, ":")
if strings.Contains(line, "sub-process exited") {
b := strings.Split(line, "time=\"")
if len(b) > 1 {
end = b[1][:19]
}
}
if len(step) < 2 || strings.Contains(line, "time=") || strings.TrimSpace(strings.Join(step[1:], " : ")) == "" || strings.TrimSpace(strings.Join(step[1:], " : ")) == a.Name {
continue
}
log := stripansi.Strip(strings.TrimSpace(strings.Join(step[1:], " : ")))
t, e := strconv.Unquote(log)
if e == nil {
logs = append(logs, t)
} else {
logs = append(logs, strings.ReplaceAll(log, "\"", "`"))
}
if strings.Contains(logs[len(logs)-1], "Error") {
err = true
}
}
return &workflow
}
func parseValue(line string) string {
parts := strings.SplitN(line, ":", 2)
if len(parts) < 2 {
return ""
status := "Pending"
if a.StepCount > 0 {
status = "Running"
}
return strings.TrimSpace(parts[1])
}
func parseBoolValue(line string) bool {
value := parseValue(line)
return value == "True"
if a.StepCount == a.StepMax {
if err {
status = "Failed"
} else {
status = "Succeeded"
}
}
duration := float64(0)
if end != "" {
timeE, _ := time.Parse("2006-01-02T15:04:05", end)
duration = timeE.Sub(a.Started).Seconds()
}
argo := &ArgoWatch{
Name: a.Name,
Namespace: a.Namespace,
Status: status,
Created: a.CreatedDate,
Started: a.Started.Format("2006-01-02 15:04:05"),
Conditions: Conditions{
PodRunning: a.StepCount > 0 && a.StepCount < a.StepMax,
Completed: a.StepCount == a.StepMax,
},
Progress: fmt.Sprintf("%v/%v", a.StepCount, a.StepMax),
Duration: fmt.Sprintf("%v", fmt.Sprintf("%.2f", duration)+"s"),
Logs: logs,
}
if !argo.Completed {
a.StartStepRecording()
}
return argo
}

34
models/services.go
View File

@@ -1,40 +1,36 @@
package models
type ServiceResource struct {
Action string `yaml:"action,omitempty"`
SuccessCondition string `yaml:"successCondition,omitempty"`
SuccessCondition string `yaml:"successCondition,omitempty"`
FailureCondition string `yaml:"failureCondition,omitempty"`
SetOwnerReference bool `yaml:"setOwnerReference,omitempty"`
Manifest string `yaml:"manifest,omitempty"`
}
type Service struct {
APIVersion string `yaml:"apiVersion"`
Kind string `yaml:"kind"`
Metadata Metadata `yaml:"metadata"`
Spec ServiceSpec `yaml:"spec"`
APIVersion string `yaml:"apiVersion"`
Kind string `yaml:"kind"`
Metadata Metadata `yaml:"metadata"`
Spec ServiceSpec `yaml:"spec"`
}
type Metadata struct {
Name string `yaml:"name"`
Name string `yaml:"name"`
}
// ServiceSpec is the specification of the Kubernetes Service
type ServiceSpec struct {
Selector map[string]string `yaml:"selector,omitempty"`
Ports []ServicePort `yaml:"ports"`
ClusterIP string `yaml:"clusterIP,omitempty"`
Type string `yaml:"type,omitempty"`
Selector map[string]string `yaml:"selector,omitempty"`
Ports []ServicePort `yaml:"ports"`
ClusterIP string `yaml:"clusterIP,omitempty"`
Type string `yaml:"type,omitempty"`
}
// ServicePort defines a port for a Kubernetes Service
type ServicePort struct {
Name string `yaml:"name"` // Even if empty need to be in the yaml
Protocol string `yaml:"protocol,omitempty"`
Port int64 `yaml:"port"`
TargetPort int64 `yaml:"targetPort,omitempty"`
NodePort int64 `yaml:"nodePort,omitempty"`
}
Name string `yaml:"name"` // Even if empty need to be in the yaml
Protocol string `yaml:"protocol,omitempty"`
Port int `yaml:"port"`
TargetPort int `yaml:"targetPort,omitempty"`
}

75
models/template.go
View File

@@ -1,5 +1,12 @@
package models
import (
"strings"
"cloud.o-forge.io/core/oc-lib/models/resources/processing"
"cloud.o-forge.io/core/oc-lib/models/resources/storage"
)
type Parameter struct {
Name string `yaml:"name,omitempty"`
Value string `yaml:"value,omitempty"`
@@ -12,9 +19,28 @@ type Container struct {
VolumeMounts []VolumeMount `yaml:"volumeMounts,omitempty"`
}
func (c *Container) AddVolumeMount(volumeMount VolumeMount, volumes []VolumeMount) []VolumeMount {
for _, vm := range c.VolumeMounts {
if vm.Name == volumeMount.Name {
return volumes
}
}
c.VolumeMounts = append(c.VolumeMounts, volumeMount)
for _, vm := range c.VolumeMounts {
for _, v := range volumes {
if vm.Name == v.Name {
return volumes
}
}
}
volumes = append(volumes, volumeMount)
return volumes
}
type VolumeMount struct {
Name string `yaml:"name"`
MountPath string `yaml:"mountPath"`
Name string `yaml:"name"`
MountPath string `yaml:"mountPath"`
Storage *storage.StorageResource `yaml:"-"`
}
type Task struct {
@@ -39,8 +65,47 @@ type Template struct {
Inputs struct {
Parameters []Parameter `yaml:"parameters"`
} `yaml:"inputs,omitempty"`
Container Container `yaml:"container,omitempty"`
Dag Dag `yaml:"dag,omitempty"`
Container Container `yaml:"container,omitempty"`
Dag *Dag `yaml:"dag,omitempty"`
Metadata TemplateMetadata `yaml:"metadata,omitempty"`
Resource ServiceResource `yaml:"resource,omitempty"`
Resource ServiceResource `yaml:"resource,omitempty"`
}
func (template *Template) CreateContainer(processing *processing.ProcessingResource, dag *Dag) {
container := Container{Image: processing.Container.Image}
if container.Image == "" {
return
}
container.Command = []string{"sh", "-c"} // all is bash
for name := range processing.Container.Env {
template.Inputs.Parameters = append(template.Inputs.Parameters, Parameter{Name: name})
}
for _, a := range strings.Split(processing.Container.Args, " ") {
container.Args = append(container.Args, template.replacePerEnv(a, processing.Container.Env, dag))
}
cmd := strings.ReplaceAll(processing.Container.Command, container.Image, "")
container.Args = []string{cmd + " " + strings.Join(container.Args, " ")}
template.Container = container
}
func (template *Template) replacePerEnv(arg string, envs map[string]string, dag *Dag) string {
for k, v := range envs {
if strings.Contains(arg, k) {
value := v
for _, task := range dag.Tasks {
if task.Name == template.Name {
for _, p := range task.Arguments.Parameters {
if p.Name == k {
value = p.Value
break
}
}
}
}
arg = strings.ReplaceAll(arg, "$"+k, value)
arg = strings.ReplaceAll(arg, "${"+k+"}", value)
arg = strings.ReplaceAll(arg, k, value)
}
}
return arg
}