oc-monitord/main.go

316 lines
8.9 KiB
Go
Raw Normal View History

2024-07-04 09:14:25 +02:00
package main
import (
2024-08-06 11:40:30 +02:00
"bufio"
"encoding/json"
"fmt"
"io"
"os"
2024-08-06 11:40:30 +02:00
"os/exec"
"regexp"
"strings"
2024-08-07 17:24:09 +02:00
"sync"
"time"
"oc-monitor/conf"
2024-08-06 11:40:30 +02:00
"oc-monitor/models"
"oc-monitor/workflow_builder"
2024-08-06 11:40:30 +02:00
oclib "cloud.o-forge.io/core/oc-lib"
2024-08-07 17:24:09 +02:00
"cloud.o-forge.io/core/oc-lib/logs"
2024-08-07 17:24:09 +02:00
"cloud.o-forge.io/core/oc-lib/models/workflow_execution"
"github.com/akamensky/argparse"
2024-08-06 11:40:30 +02:00
"github.com/google/uuid"
"github.com/goraz/onion"
"github.com/rs/zerolog"
2024-07-04 09:14:25 +02:00
)
2024-08-06 11:40:30 +02:00
var logger zerolog.Logger
var wf_logger zerolog.Logger
var parser argparse.Parser
var monitorLocal bool
var workflowName string
2024-08-07 17:24:09 +02:00
const namespace = "-n argo"
const defaultConfigFile = "/etc/oc/ocmonitor_conf.json"
const localConfigFile = "./conf/ocmonitor_conf.json"
2024-07-04 09:14:25 +02:00
func main() {
2024-08-06 11:40:30 +02:00
monitorLocal = false
// Test if monitor is launched outside (with parameters) or in a k8s environment (env variables sets)
if os.Getenv("KUBERNETES_SERVICE_HOST") == ""{
// Not in a k8s environment, get conf from parameters
2024-08-06 11:40:30 +02:00
fmt.Println("Executes outside of k8s")
parser = *argparse.NewParser("oc-monitor","Launch the execution of a workflow given as a parameter and sends the produced logs to a loki database")
loadConfig(false, &parser)
2024-08-06 11:40:30 +02:00
} else {
// Executed in a k8s environment
fmt.Println("Executes inside a k8s")
monitorLocal = true
loadConfig(true,nil)
2024-07-04 09:14:25 +02:00
}
2024-08-06 11:40:30 +02:00
logs.SetAppName("oc-monitor")
logger = logs.CreateLogger("oc-monitor", conf.GetConfig().LokiURL)
2024-08-06 11:40:30 +02:00
logger.Debug().Msg("Loki URL : " + conf.GetConfig().LokiURL)
2024-08-07 17:24:09 +02:00
logger.Debug().Msg("Workflow executed : " + conf.GetConfig().ExecutionID)
2024-08-06 11:40:30 +02:00
oclib.SetConfig(conf.GetConfig().MongoUrl,conf.GetConfig().Database)
oclib.Init("oc-monitor")
2024-08-07 17:24:09 +02:00
wf_id := retrieveWorkflowId(conf.GetConfig().ExecutionID)
conf.GetConfig().WorkflowID = wf_id
// // create argo
2024-08-06 11:40:30 +02:00
new_wf := workflow_builder.WorflowDB{}
err := new_wf.LoadFrom(conf.GetConfig().WorkflowID)
if err != nil {
logger.Error().Msg("Could not retrieve workflow " + conf.GetConfig().WorkflowID + " from oc-catalog API")
}
argo_file_path, err := new_wf.ExportToArgo()
if err != nil {
logger.Error().Msg("Could not create the Argo file for " + conf.GetConfig().WorkflowID )
logger.Error().Msg(err.Error())
2024-07-04 09:14:25 +02:00
}
2024-08-06 11:40:30 +02:00
logger.Debug().Msg("Created :" + argo_file_path)
workflowName = getContainerName(argo_file_path)
2024-08-07 17:24:09 +02:00
wf_logger = logger.With().Str("argo_name", workflowName).Str("workflow_id",conf.GetConfig().WorkflowID).Str("workflow_execution_id",conf.GetConfig().ExecutionID).Logger()
2024-08-06 11:40:30 +02:00
wf_logger.Debug().Msg("Testing argo name")
executeWorkflow(argo_file_path)
2024-08-07 17:24:09 +02:00
fmt.Println("Logs sent to Loki successfully.")
}
2024-08-06 11:40:30 +02:00
2024-08-07 17:24:09 +02:00
func retrieveWorkflowId(exec_id string) string {
2024-08-06 11:40:30 +02:00
2024-08-07 17:24:09 +02:00
res := oclib.LoadOne(oclib.LibDataEnum(oclib.WORKFLOW_EXECUTION),exec_id)
2024-08-07 17:24:09 +02:00
if res.Code != 200 {
logger.Error().Msg("Could not retrieve workflow ID from execution ID " + exec_id)
return ""
}
2024-08-07 17:24:09 +02:00
wf_exec := res.ToWorkflowExecution()
2024-07-04 09:14:25 +02:00
2024-08-07 17:24:09 +02:00
return wf_exec.WorkflowID
2024-07-04 09:14:25 +02:00
}
2024-08-06 11:40:30 +02:00
func executeWorkflow(argo_file_path string) {
var stdout, stderr io.ReadCloser
2024-08-07 17:24:09 +02:00
// var stderr io.ReadCloser
2024-08-06 11:40:30 +02:00
var err error
2024-08-07 17:24:09 +02:00
cmd := exec.Command("argo", "submit", "--watch", "argo_workflows/"+argo_file_path, "--serviceaccount=argo", "-n","argo")
2024-08-06 11:40:30 +02:00
if stdout, err = cmd.StdoutPipe(); err != nil{
wf_logger.Error().Msg("Could not retrieve stdoutpipe " + err.Error())
return
}
if stderr, err = cmd.StderrPipe(); err != nil{
wf_logger.Error().Msg("Could not retrieve stderrpipe " + err.Error())
return
}
if err := cmd.Start(); err != nil {
panic(err)
}
2024-08-07 17:24:09 +02:00
var wg sync.WaitGroup
go logWorkflow(stdout, &wg)
go logWorkflow(stderr,&wg)
time.Sleep(time.Second * 1)
go logPods(workflowName)
2024-08-06 11:40:30 +02:00
if err := cmd.Wait(); err != nil {
wf_logger.Error().Msg("Could not execute argo submit")
2024-08-07 17:24:09 +02:00
wf_logger.Error().Msg(err.Error() + bufio.NewScanner(stderr).Text())
// updateStatus(exec_id, FATAL)
2024-08-06 11:40:30 +02:00
}
2024-08-07 17:24:09 +02:00
wg.Wait()
2024-08-06 11:40:30 +02:00
}
// We could improve this function by creating an object with the same attribute as the output
// and only send a new log if the current object has different values than the previous
2024-08-07 17:24:09 +02:00
func logWorkflow(pipe io.ReadCloser, wg *sync.WaitGroup) {
2024-08-06 11:40:30 +02:00
var current_watch, previous_watch models.ArgoWatch
watch_output := make([]string,0)
scanner := bufio.NewScanner(pipe)
for scanner.Scan() {
log := scanner.Text()
watch_output = append(watch_output, log)
if(strings.HasPrefix(log, "Progress:")){
current_watch = *models.NewArgoLogs(watch_output)
2024-08-07 17:24:09 +02:00
fmt.Println("Status : " + current_watch.Status)
workflowName = current_watch.Name
2024-08-06 11:40:30 +02:00
if(!current_watch.Equals(previous_watch)){
2024-08-07 17:24:09 +02:00
wg.Add(1)
defer wg.Done()
if(current_watch.Status == "Succeeded"){
fmt.Print()
}
checkStatus(current_watch.Status, previous_watch.Status)
2024-08-06 11:40:30 +02:00
jsonified, err := json.Marshal(current_watch)
if err != nil {
logger.Error().Msg("Could not create watch log")
}
wf_logger.Info().Msg(string(jsonified))
previous_watch = current_watch
2024-08-07 17:24:09 +02:00
current_watch = models.ArgoWatch{}
2024-08-06 11:40:30 +02:00
}
}
}
}
2024-08-07 17:24:09 +02:00
// Debug, no logs sent
func logPods(workflow_name string){
var stderr io.ReadCloser
var err error
cmd := exec.Command("argo","logs",workflow_name, "-n", "argo")
if stderr, err = cmd.StderrPipe(); err != nil{
wf_logger.Error().Msg("Could not retrieve stderrpipe " + err.Error())
return
}
if err := cmd.Start(); err != nil {
panic(err)
}
scanner := bufio.NewScanner(stderr)
for scanner.Scan() {
log := scanner.Text()
// fmt.Println(log)
wf_logger.Info().Msg(log)
}
if err := cmd.Wait(); err != nil {
wf_logger.Error().Msg("Could not execute argo logs")
}
}
func loadConfig(is_k8s bool, parser *argparse.Parser){
var o *onion.Onion
2024-08-06 11:40:30 +02:00
o = initOnion(o)
conf.GetConfig().OcCatalogUrl = o.GetStringDefault("oc-catalog","https://localhost:8087")
// These variables can only be retrieved in the onion
// Variables that don't depend on the environmen (from conf file), can be loaded after
// We can't use underscore in the env variable names because it's the delimitor with OCMONITOR too
setConf(is_k8s, o, parser)
2024-08-07 17:24:09 +02:00
if (!IsValidUUID(conf.GetConfig().ExecutionID)){
2024-08-06 11:40:30 +02:00
logger.Fatal().Msg("Provided ID is not an UUID")
}
}
func setConf(is_k8s bool, o *onion.Onion, parser *argparse.Parser) {
if is_k8s {
conf.GetConfig().LokiURL = o.GetStringDefault("lokiurl", "http://127.0.0.1:3100")
2024-08-07 17:24:09 +02:00
conf.GetConfig().ExecutionID = o.GetString("workflow")
conf.GetConfig().MongoUrl = o.GetStringDefault("mongourl", "mongodb://127.0.0.1:27017")
2024-08-06 11:40:30 +02:00
conf.GetConfig().Database = o.GetStringDefault("database", "DC_myDC")
} else {
url := parser.String("u", "url", &argparse.Options{Required: true, Default: "http://127.0.0.1:3100", Help: "Url to the Loki database logs will be sent to"})
2024-08-07 17:24:09 +02:00
workflow := parser.String("w", "workflow", &argparse.Options{Required: true, Help: "Execution ID of the workflow to request from oc-catalog API"})
2024-08-06 11:40:30 +02:00
mongo := parser.String("m", "mongo", &argparse.Options{Required: true, Help: "URL to reach the MongoDB"})
db := parser.String("d", "database", &argparse.Options{Required: true, Help: "Name of the database to query in MongoDB"})
err := parser.Parse(os.Args)
if err != nil {
fmt.Println(parser.Usage(err))
os.Exit(1)
}
conf.GetConfig().LokiURL = *url
2024-08-07 17:24:09 +02:00
conf.GetConfig().ExecutionID = *workflow
2024-08-06 11:40:30 +02:00
conf.GetConfig().MongoUrl = *mongo
conf.GetConfig().Database = *db
}
}
func initOnion(o *onion.Onion) *onion.Onion {
logger = logs.CreateLogger("oc-monitor", "")
configFile := ""
l3 := onion.NewEnvLayerPrefix("_", "OCMONITOR")
l2, err := onion.NewFileLayer(defaultConfigFile, nil)
if err == nil {
logger.Info().Msg("Config file found : " + defaultConfigFile)
configFile = defaultConfigFile
}
l1, err := onion.NewFileLayer(localConfigFile, nil)
if err == nil {
logger.Info().Msg("Local config file found " + localConfigFile + ", overriding default file")
configFile = localConfigFile
}
if configFile == "" {
logger.Info().Msg("No config file found, using env")
o = onion.New(l3)
} else if l1 == nil && l2 == nil {
o = onion.New(l1, l2, l3)
} else if l1 == nil {
o = onion.New(l2, l3)
} else if l2 == nil {
o = onion.New(l1, l3)
}
2024-08-06 11:40:30 +02:00
return o
}
2024-08-06 11:40:30 +02:00
func IsValidUUID(u string) bool {
_, err := uuid.Parse(u)
return err == nil
}
2024-08-06 11:40:30 +02:00
func getContainerName(argo_file string) string {
regex := "([a-zA-Z]+-[a-zA-Z]+)"
re := regexp.MustCompile(regex)
2024-08-06 11:40:30 +02:00
container_name := re.FindString(argo_file)
return container_name
}
2024-08-07 17:24:09 +02:00
// Uses the ArgoWatch object to update status of the workflow execution object
func checkStatus(current string, previous string) {
if current != previous {
updateStatus(current)
}
}
func updateStatus(status string) {
exec_id := conf.GetConfig().ExecutionID
wf_exec := &workflow_execution.WorkflowExecution{}
wf_exec.ArgoStatusToState(status)
serialized := wf_exec.Serialize()
res := oclib.UpdateOne(oclib.LibDataEnum(oclib.WORKFLOW_EXECUTION),serialized, exec_id)
if res.Code != 200 {
logger.Error().Msg("Could not update status for workflow execution " + exec_id)
}
fmt.Printf("status argo : %s /nstatus db : %s",status,serialized["state"])
}