package main import ( "bufio" "encoding/json" "fmt" "io" "log" "os" "os/exec" "regexp" "strings" "oc-monitor/conf" "oc-monitor/models" "oc-monitor/workflow_builder" oclib "cloud.o-forge.io/core/oc-lib" "cloud.o-forge.io/core/oc-lib/logs" "github.com/akamensky/argparse" "github.com/google/uuid" "github.com/goraz/onion" "github.com/rs/zerolog" ) var logger zerolog.Logger var wf_logger zerolog.Logger var parser argparse.Parser var monitorLocal bool var workflowName string const defaultConfigFile = "/etc/oc/ocmonitor_conf.json" const localConfigFile = "./conf/ocmonitor_conf.json" func main() { monitorLocal = false // Test if monitor is launched outside (with parameters) or in a k8s environment (env variables sets) if os.Getenv("KUBERNETES_SERVICE_HOST") == ""{ // Not in a k8s environment, get conf from parameters fmt.Println("Executes outside of k8s") parser = *argparse.NewParser("oc-monitor","Launch the execution of a workflow given as a parameter and sends the produced logs to a loki database") loadConfig(false, &parser) } else { // Executed in a k8s environment fmt.Println("Executes inside a k8s") monitorLocal = true loadConfig(true,nil) } logs.SetAppName("oc-monitor") logger = logs.CreateLogger("oc-monitor", conf.GetConfig().LokiURL) logger.Debug().Msg("Loki URL : " + conf.GetConfig().LokiURL) logger.Debug().Msg("Workflow executed : " + conf.GetConfig().WorkflowID) oclib.SetConfig(conf.GetConfig().MongoUrl,conf.GetConfig().Database) oclib.Init("oc-monitor") // create argo new_wf := workflow_builder.WorflowDB{} err := new_wf.LoadFrom(conf.GetConfig().WorkflowID) if err != nil { logger.Error().Msg("Could not retrieve workflow " + conf.GetConfig().WorkflowID + " from oc-catalog API") } argo_file_path, err := new_wf.ExportToArgo() if err != nil { logger.Error().Msg("Could not create the Argo file for " + conf.GetConfig().WorkflowID ) logger.Error().Msg(err.Error()) } logger.Debug().Msg("Created :" + argo_file_path) workflowName = getContainerName(argo_file_path) wf_logger = logger.With().Str("argo_name", workflowName).Str("workflow_id",conf.GetConfig().WorkflowID).Logger() wf_logger.Debug().Msg("Testing argo name") executeWorkflow(argo_file_path) // logger.Info().Msg(string(output)) // // Initialize LokiLogger // lokiLogger := NewLokiLogger("http://localhost:3100/loki/api/v1/push") // Replace with your Loki URL // Run the Argo command // logger.Info().Msg(string(output)) // // Send logs to Loki // if err := lokiLogger.Log(`{job="argo"}`, string(output)); err != nil { // log.Fatalf("failed to send logs to Loki: %v", err) // } log.Println("Logs sent to Loki successfully.") } func executeWorkflow(argo_file_path string) { var stdout, stderr io.ReadCloser var err error cmd := exec.Command("argo", "submit", "--watch", "argo_workflows/"+argo_file_path, "--serviceaccount=argo", "-n", "argo") if stdout, err = cmd.StdoutPipe(); err != nil{ wf_logger.Error().Msg("Could not retrieve stdoutpipe " + err.Error()) return } if stderr, err = cmd.StderrPipe(); err != nil{ wf_logger.Error().Msg("Could not retrieve stderrpipe " + err.Error()) return } if err := cmd.Start(); err != nil { panic(err) } go logWorkflow(stdout) go logWorkflow(stderr) if err := cmd.Wait(); err != nil { wf_logger.Error().Msg("Could not execute argo submit") } // Function to log the logs generated by the pods // logPods(need to retrieve the name of the workflow) { // executes "argo logs nameWorkflow" // returns the output to a logging method //} } // We could improve this function by creating an object with the same attribute as the output // and only send a new log if the current object has different values than the previous func logWorkflow(pipe io.ReadCloser) { var current_watch, previous_watch models.ArgoWatch watch_output := make([]string,0) scanner := bufio.NewScanner(pipe) for scanner.Scan() { log := scanner.Text() watch_output = append(watch_output, log) if(strings.HasPrefix(log, "Progress:")){ current_watch = *models.NewArgoLogs(watch_output) if(!current_watch.Equals(previous_watch)){ jsonified, err := json.Marshal(current_watch) if err != nil { logger.Error().Msg("Could not create watch log") } wf_logger.Info().Msg(string(jsonified)) previous_watch = current_watch } } } } func loadConfig(is_k8s bool, parser *argparse.Parser){ var o *onion.Onion o = initOnion(o) conf.GetConfig().OcCatalogUrl = o.GetStringDefault("oc-catalog","https://localhost:8087") // These variables can only be retrieved in the onion // Variables that don't depend on the environmen (from conf file), can be loaded after // We can't use underscore in the env variable names because it's the delimitor with OCMONITOR too setConf(is_k8s, o, parser) if (!IsValidUUID(conf.GetConfig().WorkflowID)){ logger.Fatal().Msg("Provided ID is not an UUID") } } func setConf(is_k8s bool, o *onion.Onion, parser *argparse.Parser) { if is_k8s { conf.GetConfig().LokiURL = o.GetStringDefault("lokiurl", "http://127.0.0.1:3100") conf.GetConfig().WorkflowID = o.GetString("workflow") conf.GetConfig().LokiURL = o.GetStringDefault("mongourl", "mongodb://127.0.0.1:27017") conf.GetConfig().Database = o.GetStringDefault("database", "DC_myDC") } else { url := parser.String("u", "url", &argparse.Options{Required: true, Default: "http://127.0.0.1:3100", Help: "Url to the Loki database logs will be sent to"}) workflow := parser.String("w", "workflow", &argparse.Options{Required: true, Help: "Name of the workflow to request from oc-catalog API"}) mongo := parser.String("m", "mongo", &argparse.Options{Required: true, Help: "URL to reach the MongoDB"}) db := parser.String("d", "database", &argparse.Options{Required: true, Help: "Name of the database to query in MongoDB"}) err := parser.Parse(os.Args) if err != nil { fmt.Println(parser.Usage(err)) os.Exit(1) } conf.GetConfig().LokiURL = *url conf.GetConfig().WorkflowID = *workflow conf.GetConfig().MongoUrl = *mongo conf.GetConfig().Database = *db } } func initOnion(o *onion.Onion) *onion.Onion { logger = logs.CreateLogger("oc-monitor", "") configFile := "" l3 := onion.NewEnvLayerPrefix("_", "OCMONITOR") l2, err := onion.NewFileLayer(defaultConfigFile, nil) if err == nil { logger.Info().Msg("Config file found : " + defaultConfigFile) configFile = defaultConfigFile } l1, err := onion.NewFileLayer(localConfigFile, nil) if err == nil { logger.Info().Msg("Local config file found " + localConfigFile + ", overriding default file") configFile = localConfigFile } if configFile == "" { logger.Info().Msg("No config file found, using env") o = onion.New(l3) } else if l1 == nil && l2 == nil { o = onion.New(l1, l2, l3) } else if l1 == nil { o = onion.New(l2, l3) } else if l2 == nil { o = onion.New(l1, l3) } return o } func IsValidUUID(u string) bool { _, err := uuid.Parse(u) return err == nil } func getContainerName(argo_file string) string { regex := "([a-zA-Z]+-[a-zA-Z]+)" re := regexp.MustCompile(regex) container_name := re.FindString(argo_file) return container_name }