Datacenter no more handle booking but is fully charged with Kube & minio allocate per NATS
This commit is contained in:
297
infrastructure/minio/minio_setter.go
Normal file
297
infrastructure/minio/minio_setter.go
Normal file
@@ -0,0 +1,297 @@
|
||||
package minio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
|
||||
"oc-datacenter/conf"
|
||||
|
||||
oclib "cloud.o-forge.io/core/oc-lib"
|
||||
"cloud.o-forge.io/core/oc-lib/models/live"
|
||||
"cloud.o-forge.io/core/oc-lib/tools"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
// MinioCredentialEvent is the NATS payload used to transfer Minio credentials between peers.
|
||||
//
|
||||
// Two-phase protocol over PROPALGATION_EVENT (Action = PB_MINIO_CONFIG):
|
||||
// - Phase 1 – role assignment (Access == ""):
|
||||
// oc-discovery routes this to the SOURCE peer (Minio host) → InitializeAsSource.
|
||||
// - Phase 2 – credential delivery (Access != ""):
|
||||
// oc-discovery routes this to the TARGET peer (compute host) → InitializeAsTarget.
|
||||
type MinioCredentialEvent struct {
|
||||
ExecutionsID string `json:"executions_id"`
|
||||
MinioID string `json:"minio_id"`
|
||||
Access string `json:"access"`
|
||||
Secret string `json:"secret"`
|
||||
SourcePeerID string `json:"source_peer_id"`
|
||||
DestPeerID string `json:"dest_peer_id"`
|
||||
URL string `json:"url"`
|
||||
// OriginID is the peer that initiated the provisioning request.
|
||||
// The PB_CONSIDERS response is routed back to this peer.
|
||||
OriginID string `json:"origin_id"`
|
||||
}
|
||||
|
||||
// minioConsidersPayload is the PB_CONSIDERS payload emitted after minio provisioning.
|
||||
type minioConsidersPayload struct {
|
||||
OriginID string `json:"origin_id"`
|
||||
ExecutionsID string `json:"executions_id"`
|
||||
Secret string `json:"secret,omitempty"`
|
||||
Error *string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// emitConsiders publishes a PB_CONSIDERS back to OriginID with the result of
|
||||
// the minio provisioning. secret is the provisioned credential; err is nil on success.
|
||||
func emitConsiders(executionsID, originID, secret string, provErr error) {
|
||||
var errStr *string
|
||||
if provErr != nil {
|
||||
s := provErr.Error()
|
||||
errStr = &s
|
||||
}
|
||||
payload, _ := json.Marshal(minioConsidersPayload{
|
||||
OriginID: originID,
|
||||
ExecutionsID: executionsID,
|
||||
Secret: secret,
|
||||
Error: errStr,
|
||||
})
|
||||
b, _ := json.Marshal(&tools.PropalgationMessage{
|
||||
DataType: tools.STORAGE_RESOURCE.EnumIndex(),
|
||||
Action: tools.PB_CONSIDERS,
|
||||
Payload: payload,
|
||||
})
|
||||
go tools.NewNATSCaller().SetNATSPub(tools.PROPALGATION_EVENT, tools.NATSResponse{
|
||||
FromApp: "oc-datacenter",
|
||||
Datatype: -1,
|
||||
Method: int(tools.PROPALGATION_EVENT),
|
||||
Payload: b,
|
||||
})
|
||||
}
|
||||
|
||||
// MinioSetter carries the execution context for a Minio credential provisioning.
|
||||
type MinioSetter struct {
|
||||
ExecutionsID string // used as both the bucket name and the K8s namespace suffix
|
||||
MinioID string // ID of the Minio storage resource
|
||||
}
|
||||
|
||||
func NewMinioSetter(execID, minioID string) *MinioSetter {
|
||||
return &MinioSetter{ExecutionsID: execID, MinioID: minioID}
|
||||
}
|
||||
|
||||
// InitializeAsSource is called on the peer that hosts the Minio instance.
|
||||
//
|
||||
// It:
|
||||
// 1. Looks up the live-storage endpoint URL for MinioID.
|
||||
// 2. Creates a scoped service account (access + secret limited to the execution bucket).
|
||||
// 3. Creates the execution bucket.
|
||||
// 4. If source and dest are the same peer, calls InitializeAsTarget directly.
|
||||
// Otherwise, publishes a MinioCredentialEvent via NATS (Phase 2) so that
|
||||
// oc-discovery can route the credentials to the compute peer.
|
||||
func (m *MinioSetter) InitializeAsSource(ctx context.Context, localPeerID, destPeerID, originID string) {
|
||||
logger := oclib.GetLogger()
|
||||
|
||||
url, err := m.loadMinioURL(localPeerID)
|
||||
if err != nil {
|
||||
logger.Error().Msg("MinioSetter.InitializeAsSource: " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
service := NewMinioService(url)
|
||||
if err := service.CreateClient(); err != nil {
|
||||
logger.Error().Msg("MinioSetter.InitializeAsSource: failed to create admin client: " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
access, secret, err := service.CreateCredentials(m.ExecutionsID)
|
||||
if err != nil {
|
||||
logger.Error().Msg("MinioSetter.InitializeAsSource: failed to create service account: " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if err := service.CreateBucket(m.MinioID, m.ExecutionsID); err != nil {
|
||||
logger.Error().Msg("MinioSetter.InitializeAsSource: failed to create bucket: " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
logger.Info().Msg("MinioSetter.InitializeAsSource: bucket and service account ready for " + m.ExecutionsID)
|
||||
|
||||
event := MinioCredentialEvent{
|
||||
ExecutionsID: m.ExecutionsID,
|
||||
MinioID: m.MinioID,
|
||||
Access: access,
|
||||
Secret: secret,
|
||||
SourcePeerID: localPeerID,
|
||||
DestPeerID: destPeerID,
|
||||
OriginID: originID,
|
||||
}
|
||||
|
||||
if destPeerID == localPeerID {
|
||||
// Same peer: store the secret locally without going through NATS.
|
||||
m.InitializeAsTarget(ctx, event)
|
||||
return
|
||||
}
|
||||
|
||||
// Cross-peer: publish credentials (Phase 2) so oc-discovery routes them to the compute peer.
|
||||
payload, err := json.Marshal(event)
|
||||
if err != nil {
|
||||
logger.Error().Msg("MinioSetter.InitializeAsSource: failed to marshal credential event: " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if b, err := json.Marshal(&tools.PropalgationMessage{
|
||||
DataType: -1,
|
||||
Action: tools.PB_MINIO_CONFIG,
|
||||
Payload: payload,
|
||||
}); err == nil {
|
||||
go tools.NewNATSCaller().SetNATSPub(tools.PROPALGATION_EVENT, tools.NATSResponse{
|
||||
FromApp: "oc-datacenter",
|
||||
Datatype: -1,
|
||||
User: "",
|
||||
Method: int(tools.PROPALGATION_EVENT),
|
||||
Payload: b,
|
||||
})
|
||||
logger.Info().Msg("MinioSetter.InitializeAsSource: credentials published via NATS for " + m.ExecutionsID)
|
||||
}
|
||||
}
|
||||
|
||||
// InitializeAsTarget is called on the peer that runs the compute workload.
|
||||
//
|
||||
// It stores the Minio credentials received from the source peer (via NATS or directly)
|
||||
// as a Kubernetes secret inside the execution namespace, making them available to pods.
|
||||
func (m *MinioSetter) InitializeAsTarget(ctx context.Context, event MinioCredentialEvent) {
|
||||
logger := oclib.GetLogger()
|
||||
|
||||
k, err := tools.NewKubernetesService(
|
||||
conf.GetConfig().KubeHost+":"+conf.GetConfig().KubePort,
|
||||
conf.GetConfig().KubeCA, conf.GetConfig().KubeCert, conf.GetConfig().KubeData,
|
||||
)
|
||||
if err != nil {
|
||||
logger.Error().Msg("MinioSetter.InitializeAsTarget: failed to create k8s service: " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if err := k.CreateSecret(ctx, event.MinioID, event.ExecutionsID, event.Access, event.Secret); err != nil {
|
||||
logger.Error().Msg("MinioSetter.InitializeAsTarget: failed to create k8s secret: " + err.Error())
|
||||
emitConsiders(event.ExecutionsID, event.OriginID, "", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := NewMinioService(event.URL).CreateMinioConfigMap(event.MinioID, event.ExecutionsID, event.URL); err == nil {
|
||||
logger.Error().Msg("MinioSetter.InitializeAsTarget: failed to create config map: " + err.Error())
|
||||
emitConsiders(event.ExecutionsID, event.OriginID, "", err)
|
||||
return
|
||||
}
|
||||
|
||||
logger.Info().Msg("MinioSetter.InitializeAsTarget: Minio credentials stored in namespace " + event.ExecutionsID)
|
||||
emitConsiders(event.ExecutionsID, event.OriginID, event.Secret, nil)
|
||||
}
|
||||
|
||||
// MinioDeleteEvent is the NATS payload used to tear down Minio resources.
|
||||
// It mirrors MinioCredentialEvent but carries the access key for revocation.
|
||||
type MinioDeleteEvent struct {
|
||||
ExecutionsID string `json:"executions_id"`
|
||||
MinioID string `json:"minio_id"`
|
||||
Access string `json:"access"` // service account access key to revoke on the Minio host
|
||||
SourcePeerID string `json:"source_peer_id"`
|
||||
DestPeerID string `json:"dest_peer_id"`
|
||||
OriginID string `json:"origin_id"`
|
||||
}
|
||||
|
||||
// TeardownAsTarget is called on the peer that runs the compute workload.
|
||||
// It reads the stored access key from the K8s secret, then removes both the secret
|
||||
// and the artifact-repository ConfigMap from the execution namespace.
|
||||
// For same-peer deployments it calls TeardownAsSource directly; otherwise it
|
||||
// publishes a MinioDeleteEvent via NATS (PB_DELETE) so oc-discovery routes it to
|
||||
// the Minio host peer.
|
||||
func (m *MinioSetter) TeardownAsTarget(ctx context.Context, event MinioDeleteEvent) {
|
||||
logger := oclib.GetLogger()
|
||||
|
||||
k, err := tools.NewKubernetesService(
|
||||
conf.GetConfig().KubeHost+":"+conf.GetConfig().KubePort,
|
||||
conf.GetConfig().KubeCA, conf.GetConfig().KubeCert, conf.GetConfig().KubeData,
|
||||
)
|
||||
if err != nil {
|
||||
logger.Error().Msg("MinioSetter.TeardownAsTarget: failed to create k8s service: " + err.Error())
|
||||
emitConsiders(event.ExecutionsID, event.OriginID, "", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Read the access key from the K8s secret before deleting it.
|
||||
accessKey := event.Access
|
||||
if accessKey == "" {
|
||||
if secret, err := k.Set.CoreV1().Secrets(event.ExecutionsID).Get(
|
||||
ctx, event.MinioID+"-secret-s3", metav1.GetOptions{},
|
||||
); err == nil {
|
||||
accessKey = string(secret.Data["access-key"])
|
||||
}
|
||||
}
|
||||
|
||||
// Delete K8s credentials secret.
|
||||
if err := k.Set.CoreV1().Secrets(event.ExecutionsID).Delete(
|
||||
ctx, event.MinioID+"-secret-s3", metav1.DeleteOptions{},
|
||||
); err != nil {
|
||||
logger.Error().Msg("MinioSetter.TeardownAsTarget: failed to delete secret: " + err.Error())
|
||||
}
|
||||
|
||||
// Delete artifact-repository ConfigMap.
|
||||
if err := NewMinioService("").DeleteMinioConfigMap(event.MinioID, event.ExecutionsID); err != nil {
|
||||
logger.Error().Msg("MinioSetter.TeardownAsTarget: failed to delete configmap: " + err.Error())
|
||||
}
|
||||
|
||||
logger.Info().Msg("MinioSetter.TeardownAsTarget: K8s resources removed for " + event.ExecutionsID)
|
||||
|
||||
// For same-peer deployments the source cleanup runs directly here so the
|
||||
// caller (REMOVE_EXECUTION handler) doesn't have to distinguish roles.
|
||||
if event.SourcePeerID == event.DestPeerID {
|
||||
event.Access = accessKey
|
||||
m.TeardownAsSource(ctx, event)
|
||||
}
|
||||
}
|
||||
|
||||
// TeardownAsSource is called on the peer that hosts the Minio instance.
|
||||
// It revokes the scoped service account and removes the execution bucket.
|
||||
func (m *MinioSetter) TeardownAsSource(ctx context.Context, event MinioDeleteEvent) {
|
||||
logger := oclib.GetLogger()
|
||||
|
||||
url, err := m.loadMinioURL(event.SourcePeerID)
|
||||
if err != nil {
|
||||
logger.Error().Msg("MinioSetter.TeardownAsSource: " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
svc := NewMinioService(url)
|
||||
if err := svc.CreateClient(); err != nil {
|
||||
logger.Error().Msg("MinioSetter.TeardownAsSource: failed to create admin client: " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if event.Access != "" {
|
||||
if err := svc.DeleteCredentials(event.Access); err != nil {
|
||||
logger.Error().Msg("MinioSetter.TeardownAsSource: failed to delete service account: " + err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
if err := svc.DeleteBucket(event.MinioID, event.ExecutionsID); err != nil {
|
||||
logger.Error().Msg("MinioSetter.TeardownAsSource: failed to delete bucket: " + err.Error())
|
||||
}
|
||||
|
||||
logger.Info().Msg("MinioSetter.TeardownAsSource: Minio resources removed for " + event.ExecutionsID)
|
||||
}
|
||||
|
||||
// loadMinioURL searches through all live storages accessible by peerID to find
|
||||
// the one that references MinioID, and returns its endpoint URL.
|
||||
func (m *MinioSetter) loadMinioURL(peerID string) (string, error) {
|
||||
res := oclib.NewRequest(oclib.LibDataEnum(oclib.LIVE_STORAGE), "", peerID, []string{}, nil).LoadAll(false)
|
||||
if res.Err != "" {
|
||||
return "", fmt.Errorf("loadMinioURL: failed to load live storages: %s", res.Err)
|
||||
}
|
||||
for _, dbo := range res.Data {
|
||||
l := dbo.(*live.LiveStorage)
|
||||
if slices.Contains(l.ResourcesID, m.MinioID) {
|
||||
return l.Source, nil
|
||||
}
|
||||
|
||||
}
|
||||
return "", fmt.Errorf("loadMinioURL: no live storage found for minio ID %s", m.MinioID)
|
||||
}
|
||||
Reference in New Issue
Block a user