From fba1608edb70e4d4e91c36cfca59f011e72c60d3 Mon Sep 17 00:00:00 2001 From: ycc Date: Mon, 2 Sep 2024 15:21:16 +0200 Subject: [PATCH 1/8] getconfig fix --- tools/conf_loader.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/conf_loader.go b/tools/conf_loader.go index 21c183d..78bf595 100644 --- a/tools/conf_loader.go +++ b/tools/conf_loader.go @@ -21,8 +21,9 @@ import ( // The configuration loader will give priority to the environment variables // The configuration loader will give priority to the local file over the default file -func GetConfLoader(AppName string) *onion.Onion { +func GetConfLoader() *onion.Onion { logger := logs.GetLogger() + AppName := logs.GetAppName() EnvPrefix := strings.ToUpper(AppName[0:2]+AppName[3:]) + "_" defaultConfigFile := "/etc/oc/" + AppName[0:2] + ".json" localConfigFile := "./" + AppName[0:2] + ".json" From bb36ac0fb4a63b3567648e9cafffd79deeee6f93 Mon Sep 17 00:00:00 2001 From: ycc Date: Wed, 4 Sep 2024 10:53:12 +0200 Subject: [PATCH 2/8] Refactor and doc --- README.md | 56 ++++++++++++++++++++++++++++++++ config/app.go | 19 +++++++++++ {tools => config}/conf.go | 10 ++++-- {tools => config}/conf_loader.go | 36 ++++++++++---------- entrypoint.go | 53 ++++++++++++++++++++++++++---- logs/logger.go | 16 +++------ models/utils/abstracts.go | 6 ++-- tools/api.go | 5 +-- tools/nats_caller.go | 5 +-- 9 files changed, 161 insertions(+), 45 deletions(-) create mode 100644 config/app.go rename {tools => config}/conf.go (75%) rename {tools => config}/conf_loader.go (56%) diff --git a/README.md b/README.md index 82d01f7..05c6e78 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,61 @@ # oc-lib +oc-lib allows read/write/search operations into the main OpenCloud databases. + +It also provides common initialization and configuration utilities for all OpenCloud components + +## Usage example in a beego API + +```go +const appname = "oc-mycomponent" + +func main() { + // Init the oc-lib + oclib.Init(appname) + + // Load the right config file + + /* The configuration loader will load the configuration from the following sources: + * - the environment variables with the prefix APPNAME_ + * - the file /etc/oc/appname.json + * - the file ./appname.json + * The configuration loader will merge the configuration from the different sources + * The configuration loader will give priority to the environment variables + * The configuration loader will give priority to the local file over the default file + */ + o := oclib.GetConfLoader() + + // init the local config object + models.GetConfig().Port = o.GetIntDefault("port", 8080) + models.GetConfig().LokiUrl = o.GetStringDefault("lokiurl", "") + models.GetConfig().LogLevel = o.GetStringDefault("loglevel", "info") + models.GetConfig().MongoUrl = o.GetStringDefault("mongourl", "mongodb://127.0.0.1:27017") + models.GetConfig().MongoDatabase = o.GetStringDefault("mongodatabase", "myDb") + models.GetConfig().NatsUrl = o.GetStringDefault("natsurl", "nats://localhost:4222") + + models.GetConfig().mycomponentparam1 = o.GetStringDefault("mycomponentparam1", "mycomponentdefault1") + models.GetConfig().mycomponentparam2 = o.GetStringDefault("mycomponentparam2", "mycomponentdefault2") + + // feed the library with the loaded config, + // this will also initialize a logger available via oclib.GetLogger() + oclib.SetConfig( + models.GetConfig().MongoUrl + models.GetConfig().MongoDatabase + models.GetConfig().NatsUrl + models.GetConfig().LokiUrl + models.GetConfig().LogLevel + ) + + // Beego init + beego.BConfig.AppName = appname + beego.BConfig.Listen.HTTPPort = models.GetConfig().Port + beego.BConfig.WebConfig.DirectoryIndex = true + beego.BConfig.WebConfig.StaticDir["/swagger"] = "swagger" + + beego.Run() +} +``` + ## SPECIAL FLOWS IN OC-LIB RESUME : ### WORKFLOW AS ITS OWN WORKSPACE diff --git a/config/app.go b/config/app.go new file mode 100644 index 0000000..a660625 --- /dev/null +++ b/config/app.go @@ -0,0 +1,19 @@ +package config + +var appname string + +// logs.CreateLogger +// Create a new logger +// Parameters: +// - appname: string : the name of the application using oclib +// - url: string : the url of a loki logger, console log only if "" +// Returns: +// - zerolog.Logger : the logger that will log for the library and the app + +func SetAppName(name string) { + appname = name +} + +func GetAppName() string { + return appname +} diff --git a/tools/conf.go b/config/conf.go similarity index 75% rename from tools/conf.go rename to config/conf.go index c72a6f9..566c4a3 100644 --- a/tools/conf.go +++ b/config/conf.go @@ -1,4 +1,4 @@ -package tools +package config import "sync" @@ -14,6 +14,8 @@ type Config struct { MongoDatabase string Host string Port string + LokiUrl string + LogLevel string } func (c Config) GetUrl() string { @@ -34,12 +36,14 @@ func GetConfig() *Config { return instance } -func SetConfig(url string, database string, natsUrl string) *Config { +func SetConfig(mongoUrl string, database string, natsUrl string, lokiUrl string, logLevel string) *Config { once.Do(func() { instance = &Config{ - MongoUrl: url, + MongoUrl: mongoUrl, MongoDatabase: database, NATSUrl: natsUrl, + LokiUrl: lokiUrl, + LogLevel: logLevel, } }) return instance diff --git a/tools/conf_loader.go b/config/conf_loader.go similarity index 56% rename from tools/conf_loader.go rename to config/conf_loader.go index 78bf595..7ca533f 100644 --- a/tools/conf_loader.go +++ b/config/conf_loader.go @@ -1,29 +1,31 @@ -package tools +package config import ( + "os" "strings" - "cloud.o-forge.io/core/oc-lib/logs" "github.com/goraz/onion" + "github.com/rs/zerolog" ) -// GetConfLoader -// Get the configuration loader for the application -// Parameters: -// - AppName: string : the name of the application -// Returns: -// - *onion.Onion : the configuration loader -// The configuration loader will load the configuration from the following sources: -// - the environment variables with the prefix APPNAME_ -// - the file /etc/oc/appname.json -// - the file ./appname.json -// The configuration loader will merge the configuration from the different sources -// The configuration loader will give priority to the environment variables -// The configuration loader will give priority to the local file over the default file +/* GetConfLoader +* Get the configuration loader for the application +* Parameters: +* - AppName: string : the name of the application +* Returns: +* - *onion.Onion : the configuration loader +* The configuration loader will load the configuration from the following sources: +* - the environment variables with the prefix APPNAME_ +* - the file /etc/oc/appname.json +* - the file ./appname.json +* The configuration loader will merge the configuration from the different sources +* The configuration loader will give priority to the environment variables +* The configuration loader will give priority to the local file over the default file + */ func GetConfLoader() *onion.Onion { - logger := logs.GetLogger() - AppName := logs.GetAppName() + logger := zerolog.New(os.Stdout).With().Timestamp().Logger() + AppName := GetAppName() EnvPrefix := strings.ToUpper(AppName[0:2]+AppName[3:]) + "_" defaultConfigFile := "/etc/oc/" + AppName[0:2] + ".json" localConfigFile := "./" + AppName[0:2] + ".json" diff --git a/entrypoint.go b/entrypoint.go index fe938fb..b463ed8 100644 --- a/entrypoint.go +++ b/entrypoint.go @@ -6,6 +6,7 @@ import ( "runtime/debug" + "cloud.o-forge.io/core/oc-lib/config" "cloud.o-forge.io/core/oc-lib/dbs" "cloud.o-forge.io/core/oc-lib/dbs/mongo" "cloud.o-forge.io/core/oc-lib/logs" @@ -24,6 +25,7 @@ import ( shared_workspace "cloud.o-forge.io/core/oc-lib/models/workspace/shared" "cloud.o-forge.io/core/oc-lib/models/workspace/shared/rules/rule" "cloud.o-forge.io/core/oc-lib/tools" + "github.com/goraz/onion" "github.com/rs/zerolog" ) @@ -95,18 +97,17 @@ func AddPath(collection LibDataEnum, path string) { paths[collection] = path } -func Init(appName string, hostname string, port string) { +func Init(appName string) { defer func() { if r := recover(); r != nil { tools.UncatchedError = append(tools.UncatchedError, errors.New("Panic recovered in Init : "+fmt.Sprintf("%v", r)+" - "+string(debug.Stack()))) fmt.Printf("Panic recovered in Init : %v - %v\n", r, string(debug.Stack())) } }() - logs.SetAppName(appName) // set the app name to the logger to define the main log chan - logs.SetLogger(logs.CreateLogger("main", "")) // create the logger - tools.GetConfig().Host = hostname // set the hostname to the config for inner discovery purpose actually not used - tools.GetConfig().Port = port // set the port to the config for inner discovery purpose actually not used - mongo.MONGOService.Init(models.GetModelsNames(), tools.GetConfig()) // init the mongo service + config.SetAppName(appName) // set the app name to the logger to define the main log chan + // create a temporary console logger for init + logs.SetLogger(logs.CreateLogger("main")) + mongo.MONGOService.Init(models.GetModelsNames(), config.GetConfig()) // init the mongo service /* Here we will check if the resource model is already stored in the database If not we will store it @@ -145,11 +146,49 @@ func Init(appName string, hostname string, port string) { } } -// GetLogger returns the main logger +// +// Expose subpackages +// + +/* GetLogger returns the main logger +* @return zerolog.Logger + */ func GetLogger() zerolog.Logger { return logs.GetLogger() } +/* SetConfig will set the config and create a logger according to app configuration +* @param url string +* @param database string +* @param natsUrl string +* @param lokiUrl string +* @param logLevel string +* @return *Config + */ +func SetConfig(mongoUrl string, database string, natsUrl string, lokiUrl string, logLevel string) *config.Config { + logs.CreateLogger("main") + return config.SetConfig(mongoUrl, database, natsUrl, lokiUrl, logLevel) +} + +/* GetConfLoader +* Get the configuration loader for the application +* Parameters: +* - AppName: string : the name of the application +* Returns: +* - *onion.Onion : the configuration loader +* The configuration loader will load the configuration from the following sources: +* - the environment variables with the prefix APPNAME_ +* - the file /etc/oc/appname.json +* - the file ./appname.json +* The configuration loader will merge the configuration from the different sources +* The configuration loader will give priority to the environment variables +* The configuration loader will give priority to the local file over the default file + */ + +func GetConfLoader() *onion.Onion { + return config.GetConfLoader() +} + /* * Search will search for the data in the database * @param filters *dbs.Filters diff --git a/logs/logger.go b/logs/logger.go index 933e3b5..0ebcbac 100644 --- a/logs/logger.go +++ b/logs/logger.go @@ -5,11 +5,12 @@ import ( "runtime" "time" + "cloud.o-forge.io/core/oc-lib/config" "github.com/rs/zerolog" ) var logger zerolog.Logger -var appname string + // logs.CreateLogger // Create a new logger // Parameters: @@ -18,14 +19,6 @@ var appname string // Returns: // - zerolog.Logger : the logger that will log for the library and the app -func SetAppName(name string) { - appname = name -} - -func GetAppName() string { - return appname -} - func GetLogger() zerolog.Logger { return logger } @@ -34,10 +27,11 @@ func SetLogger(l zerolog.Logger) { logger = l } -func CreateLogger(funcName string, url string) zerolog.Logger { +func CreateLogger(funcName string) zerolog.Logger { + url := config.GetConfig().LokiUrl if url != "" { labels := map[string]string{ - "app": appname, + "app": config.GetAppName(), "code": "go", "platform": runtime.GOOS, "function": funcName, diff --git a/models/utils/abstracts.go b/models/utils/abstracts.go index 35bf71c..2333665 100644 --- a/models/utils/abstracts.go +++ b/models/utils/abstracts.go @@ -82,9 +82,9 @@ func (dma *AbstractAccessor) GetCaller() *tools.HTTPCaller { // Init initializes the accessor with the data type and the http caller func (dma *AbstractAccessor) Init(t DataType, caller *tools.HTTPCaller) { - dma.Logger = logs.CreateLogger(t.String(), "") // Create a logger with the data type - dma.Caller = caller // Set the caller - dma.Type = t.String() // Set the data type + dma.Logger = logs.CreateLogger(t.String()) // Create a logger with the data type + dma.Caller = caller // Set the caller + dma.Type = t.String() // Set the data type } // GenericLoadOne loads one object from the database (generic) diff --git a/tools/api.go b/tools/api.go index 3c0e20c..34ba1e9 100644 --- a/tools/api.go +++ b/tools/api.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" + "cloud.o-forge.io/core/oc-lib/config" "cloud.o-forge.io/core/oc-lib/dbs/mongo" ) @@ -51,11 +52,11 @@ type API struct{} // GetState returns the state of the API func (a *API) GetState() (State, int, error) { // Check if the database is up - err := mongo.MONGOService.TestDB(GetConfig()) + err := mongo.MONGOService.TestDB(config.GetConfig()) if err != nil { return DB_FALLOUT, 200, err // If the database is not up, return database fallout } - err = mongo.MONGOService.TestCollections(GetConfig(), []string{}) // Check if the collections are up + err = mongo.MONGOService.TestCollections(config.GetConfig(), []string{}) // Check if the collections are up if err != nil { return UNPROCESSABLE_ENTITY, 200, err // If the collections are not up, return unprocessable entity } diff --git a/tools/nats_caller.go b/tools/nats_caller.go index 16806ef..d4aa7fc 100644 --- a/tools/nats_caller.go +++ b/tools/nats_caller.go @@ -4,6 +4,7 @@ import ( "encoding/json" "strings" + "cloud.o-forge.io/core/oc-lib/config" "github.com/nats-io/nats.go" ) @@ -44,10 +45,10 @@ func NewNATSCaller() *natsCaller { // SetNATSPub sets a message to the NATS server func (o *natsCaller) SetNATSPub(dataName string, method NATSMethod, data interface{}) string { - if GetConfig().NATSUrl == "" { + if config.GetConfig().NATSUrl == "" { return " -> NATS_SERVER is not set" } - nc, err := nats.Connect(GetConfig().NATSUrl) + nc, err := nats.Connect(config.GetConfig().NATSUrl) if err != nil { return " -> Could not reach NATS server : " + err.Error() } From 26cfd2a38fbb8db73c9c0db624409e4ab2d3e4b1 Mon Sep 17 00:00:00 2001 From: ycc Date: Wed, 4 Sep 2024 12:09:26 +0200 Subject: [PATCH 3/8] fix accessor init location --- entrypoint.go | 50 ++++++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/entrypoint.go b/entrypoint.go index b463ed8..6e3ed4a 100644 --- a/entrypoint.go +++ b/entrypoint.go @@ -107,6 +107,31 @@ func Init(appName string) { config.SetAppName(appName) // set the app name to the logger to define the main log chan // create a temporary console logger for init logs.SetLogger(logs.CreateLogger("main")) + +} + +// +// Expose subpackages +// + +/* GetLogger returns the main logger +* @return zerolog.Logger + */ +func GetLogger() zerolog.Logger { + return logs.GetLogger() +} + +/* SetConfig will set the config and create a logger according to app configuration and initialize mongo accessor +* @param url string +* @param database string +* @param natsUrl string +* @param lokiUrl string +* @param logLevel string +* @return *Config + */ +func SetConfig(mongoUrl string, database string, natsUrl string, lokiUrl string, logLevel string) *config.Config { + logs.CreateLogger("main") + cfg := config.SetConfig(mongoUrl, database, natsUrl, lokiUrl, logLevel) mongo.MONGOService.Init(models.GetModelsNames(), config.GetConfig()) // init the mongo service /* Here we will check if the resource model is already stored in the database @@ -144,30 +169,7 @@ func Init(appName string) { }) } } -} - -// -// Expose subpackages -// - -/* GetLogger returns the main logger -* @return zerolog.Logger - */ -func GetLogger() zerolog.Logger { - return logs.GetLogger() -} - -/* SetConfig will set the config and create a logger according to app configuration -* @param url string -* @param database string -* @param natsUrl string -* @param lokiUrl string -* @param logLevel string -* @return *Config - */ -func SetConfig(mongoUrl string, database string, natsUrl string, lokiUrl string, logLevel string) *config.Config { - logs.CreateLogger("main") - return config.SetConfig(mongoUrl, database, natsUrl, lokiUrl, logLevel) + return cfg } /* GetConfLoader From 21ca50221fbe7a3281188784f58c98127ec1c373 Mon Sep 17 00:00:00 2001 From: ycc Date: Wed, 4 Sep 2024 12:11:26 +0200 Subject: [PATCH 4/8] fix moving recover --- entrypoint.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/entrypoint.go b/entrypoint.go index 6e3ed4a..5ddd6e2 100644 --- a/entrypoint.go +++ b/entrypoint.go @@ -98,12 +98,6 @@ func AddPath(collection LibDataEnum, path string) { } func Init(appName string) { - defer func() { - if r := recover(); r != nil { - tools.UncatchedError = append(tools.UncatchedError, errors.New("Panic recovered in Init : "+fmt.Sprintf("%v", r)+" - "+string(debug.Stack()))) - fmt.Printf("Panic recovered in Init : %v - %v\n", r, string(debug.Stack())) - } - }() config.SetAppName(appName) // set the app name to the logger to define the main log chan // create a temporary console logger for init logs.SetLogger(logs.CreateLogger("main")) @@ -130,6 +124,12 @@ func GetLogger() zerolog.Logger { * @return *Config */ func SetConfig(mongoUrl string, database string, natsUrl string, lokiUrl string, logLevel string) *config.Config { + defer func() { + if r := recover(); r != nil { + tools.UncatchedError = append(tools.UncatchedError, errors.New("Panic recovered in Init : "+fmt.Sprintf("%v", r)+" - "+string(debug.Stack()))) + fmt.Printf("Panic recovered in Init : %v - %v\n", r, string(debug.Stack())) + } + }() logs.CreateLogger("main") cfg := config.SetConfig(mongoUrl, database, natsUrl, lokiUrl, logLevel) mongo.MONGOService.Init(models.GetModelsNames(), config.GetConfig()) // init the mongo service From a0cbf9ac6f4ec8621a34bcd322a5feaa4ab0ba27 Mon Sep 17 00:00:00 2001 From: ycc Date: Wed, 4 Sep 2024 14:11:08 +0200 Subject: [PATCH 5/8] conf loader bugfix --- config/conf_loader.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/conf_loader.go b/config/conf_loader.go index 7ca533f..8397953 100644 --- a/config/conf_loader.go +++ b/config/conf_loader.go @@ -27,8 +27,8 @@ func GetConfLoader() *onion.Onion { logger := zerolog.New(os.Stdout).With().Timestamp().Logger() AppName := GetAppName() EnvPrefix := strings.ToUpper(AppName[0:2]+AppName[3:]) + "_" - defaultConfigFile := "/etc/oc/" + AppName[0:2] + ".json" - localConfigFile := "./" + AppName[0:2] + ".json" + defaultConfigFile := "/etc/oc/" + AppName[3:] + ".json" + localConfigFile := "./" + AppName[3:] + ".json" var configFile string var o *onion.Onion l3 := onion.NewEnvLayerPrefix("_", EnvPrefix) From ad96b50464156c8c2b63202c1bc7b77455abbeac Mon Sep 17 00:00:00 2001 From: ycc Date: Wed, 4 Sep 2024 14:21:01 +0200 Subject: [PATCH 6/8] conf singleton issue --- config/conf.go | 11 ++++++++--- entrypoint.go | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/config/conf.go b/config/conf.go index 566c4a3..7ede833 100644 --- a/config/conf.go +++ b/config/conf.go @@ -37,7 +37,7 @@ func GetConfig() *Config { } func SetConfig(mongoUrl string, database string, natsUrl string, lokiUrl string, logLevel string) *Config { - once.Do(func() { + /*once.Do(func() { instance = &Config{ MongoUrl: mongoUrl, MongoDatabase: database, @@ -45,6 +45,11 @@ func SetConfig(mongoUrl string, database string, natsUrl string, lokiUrl string, LokiUrl: lokiUrl, LogLevel: logLevel, } - }) - return instance + })*/ + GetConfig().MongoUrl = mongoUrl + GetConfig().MongoDatabase = database + GetConfig().NATSUrl = natsUrl + GetConfig().LokiUrl = lokiUrl + GetConfig().LogLevel = logLevel + return GetConfig() } diff --git a/entrypoint.go b/entrypoint.go index 5ddd6e2..c40b5c7 100644 --- a/entrypoint.go +++ b/entrypoint.go @@ -124,6 +124,7 @@ func GetLogger() zerolog.Logger { * @return *Config */ func SetConfig(mongoUrl string, database string, natsUrl string, lokiUrl string, logLevel string) *config.Config { + cfg := config.SetConfig(mongoUrl, database, natsUrl, lokiUrl, logLevel) defer func() { if r := recover(); r != nil { tools.UncatchedError = append(tools.UncatchedError, errors.New("Panic recovered in Init : "+fmt.Sprintf("%v", r)+" - "+string(debug.Stack()))) @@ -131,7 +132,6 @@ func SetConfig(mongoUrl string, database string, natsUrl string, lokiUrl string, } }() logs.CreateLogger("main") - cfg := config.SetConfig(mongoUrl, database, natsUrl, lokiUrl, logLevel) mongo.MONGOService.Init(models.GetModelsNames(), config.GetConfig()) // init the mongo service /* Here we will check if the resource model is already stored in the database From 235c8d5ccb4116f79bdf5e00cd774265cef0da18 Mon Sep 17 00:00:00 2001 From: ycc Date: Wed, 4 Sep 2024 14:41:27 +0200 Subject: [PATCH 7/8] doc fix --- README.md | 6 +++--- entrypoint.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 05c6e78..2377796 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,9 @@ func main() { // Load the right config file /* The configuration loader will load the configuration from the following sources: - * - the environment variables with the prefix APPNAME_ - * - the file /etc/oc/appname.json - * - the file ./appname.json + * - the environment variables with the prefix OCAPPNAME_ - ex: OCMYCOMPONENT_MONGOURL + * - the file /etc/oc/appname.json - ex: /etc/oc/mycomponent.json + * - the file ./appname.json - ex: ./mycomponent.json * The configuration loader will merge the configuration from the different sources * The configuration loader will give priority to the environment variables * The configuration loader will give priority to the local file over the default file diff --git a/entrypoint.go b/entrypoint.go index c40b5c7..b620ec5 100644 --- a/entrypoint.go +++ b/entrypoint.go @@ -179,7 +179,7 @@ func SetConfig(mongoUrl string, database string, natsUrl string, lokiUrl string, * Returns: * - *onion.Onion : the configuration loader * The configuration loader will load the configuration from the following sources: -* - the environment variables with the prefix APPNAME_ +* - the environment variables with the prefix OCAPPNAME_ * - the file /etc/oc/appname.json * - the file ./appname.json * The configuration loader will merge the configuration from the different sources From 4f0ab6a3760ff092be5cfb235d8d243c74acb13a Mon Sep 17 00:00:00 2001 From: ycc Date: Wed, 4 Sep 2024 15:54:49 +0200 Subject: [PATCH 8/8] Add GetConfig to entrypoint --- entrypoint.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/entrypoint.go b/entrypoint.go index b620ec5..0834d5c 100644 --- a/entrypoint.go +++ b/entrypoint.go @@ -172,6 +172,13 @@ func SetConfig(mongoUrl string, database string, natsUrl string, lokiUrl string, return cfg } +/* GetConfig will get the config +* @return *Config + */ +func GetConfig() *config.Config { + return config.GetConfig() +} + /* GetConfLoader * Get the configuration loader for the application * Parameters: