Datacenter no more handle booking but is fully charged with Kube & minio allocate per NATS
This commit is contained in:
348
infrastructure/admiralty.go
Normal file
348
infrastructure/admiralty.go
Normal file
@@ -0,0 +1,348 @@
|
||||
package infrastructure
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"oc-datacenter/conf"
|
||||
"oc-datacenter/models"
|
||||
|
||||
oclib "cloud.o-forge.io/core/oc-lib"
|
||||
"cloud.o-forge.io/core/oc-lib/tools"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
)
|
||||
|
||||
// kubeconfigChannels holds channels waiting for kubeconfig delivery (keyed by executionID).
|
||||
var kubeconfigChannels sync.Map
|
||||
|
||||
// kubeconfigEvent is the NATS payload used to transfer the kubeconfig from the source peer to the target peer.
|
||||
type KubeconfigEvent struct {
|
||||
DestPeerID string `json:"dest_peer_id"`
|
||||
ExecutionsID string `json:"executions_id"`
|
||||
Kubeconfig string `json:"kubeconfig"`
|
||||
SourcePeerID string `json:"source_peer_id"`
|
||||
// OriginID is the peer that initiated the provisioning request.
|
||||
// The PB_CONSIDERS response is routed back to this peer.
|
||||
OriginID string `json:"origin_id"`
|
||||
}
|
||||
|
||||
// admiraltyConsidersPayload is the PB_CONSIDERS payload emitted after admiralty provisioning.
|
||||
type admiraltyConsidersPayload struct {
|
||||
OriginID string `json:"origin_id"`
|
||||
ExecutionsID string `json:"executions_id"`
|
||||
Secret string `json:"secret,omitempty"`
|
||||
Error *string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// emitAdmiraltyConsiders publishes a PB_CONSIDERS back to OriginID with the result
|
||||
// of the admiralty provisioning. secret is the base64-encoded kubeconfig; err is nil on success.
|
||||
func emitAdmiraltyConsiders(executionsID, originID, secret string, provErr error) {
|
||||
var errStr *string
|
||||
if provErr != nil {
|
||||
s := provErr.Error()
|
||||
errStr = &s
|
||||
}
|
||||
payload, _ := json.Marshal(admiraltyConsidersPayload{
|
||||
OriginID: originID,
|
||||
ExecutionsID: executionsID,
|
||||
Secret: secret,
|
||||
Error: errStr,
|
||||
})
|
||||
b, _ := json.Marshal(&tools.PropalgationMessage{
|
||||
DataType: tools.COMPUTE_RESOURCE.EnumIndex(),
|
||||
Action: tools.PB_CONSIDERS,
|
||||
Payload: payload,
|
||||
})
|
||||
go tools.NewNATSCaller().SetNATSPub(tools.PROPALGATION_EVENT, tools.NATSResponse{
|
||||
FromApp: "oc-datacenter",
|
||||
Datatype: -1,
|
||||
Method: int(tools.PROPALGATION_EVENT),
|
||||
Payload: b,
|
||||
})
|
||||
}
|
||||
|
||||
// AdmiraltySetter carries the execution context for an admiralty pairing.
|
||||
type AdmiraltySetter struct {
|
||||
ExecutionsID string // execution ID, used as the Kubernetes namespace
|
||||
NodeName string // name of the virtual node created by Admiralty on the target cluster
|
||||
}
|
||||
|
||||
func NewAdmiraltySetter(execIDS string) *AdmiraltySetter {
|
||||
return &AdmiraltySetter{
|
||||
ExecutionsID: execIDS,
|
||||
}
|
||||
}
|
||||
|
||||
// InitializeAsSource is called on the peer that acts as the SOURCE cluster (compute provider).
|
||||
// It creates the AdmiraltySource resource, generates a kubeconfig for the target peer,
|
||||
// and publishes it on NATS so the target peer can complete its side of the setup.
|
||||
func (s *AdmiraltySetter) InitializeAsSource(ctx context.Context, localPeerID string, destPeerID string, originID string) {
|
||||
logger := oclib.GetLogger()
|
||||
|
||||
serv, err := tools.NewKubernetesService(conf.GetConfig().KubeHost+":"+conf.GetConfig().KubePort,
|
||||
conf.GetConfig().KubeCA, conf.GetConfig().KubeCert, conf.GetConfig().KubeData)
|
||||
if err != nil {
|
||||
logger.Error().Msg("InitializeAsSource: failed to create service: " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Create the AdmiraltySource resource on this cluster (inlined from CreateAdmiraltySource controller)
|
||||
logger.Info().Msg("Creating AdmiraltySource ns-" + s.ExecutionsID)
|
||||
_, err = serv.CreateAdmiraltySource(ctx, s.ExecutionsID)
|
||||
if err != nil && !apierrors.IsAlreadyExists(err) {
|
||||
logger.Error().Msg("InitializeAsSource: failed to create source: " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Generate a service-account token for the namespace (inlined from GetAdmiraltyKubeconfig controller)
|
||||
token, err := serv.GenerateToken(ctx, s.ExecutionsID, 3600)
|
||||
if err != nil {
|
||||
logger.Error().Msg("InitializeAsSource: failed to generate token for ns-" + s.ExecutionsID + ": " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
kubeconfig, err := buildHostKubeWithToken(token)
|
||||
if err != nil {
|
||||
logger.Error().Msg("InitializeAsSource: " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
b, err := json.Marshal(kubeconfig)
|
||||
if err != nil {
|
||||
logger.Error().Msg("InitializeAsSource: failed to marshal kubeconfig: " + err.Error())
|
||||
return
|
||||
}
|
||||
encodedKubeconfig := base64.StdEncoding.EncodeToString(b)
|
||||
kube := KubeconfigEvent{
|
||||
ExecutionsID: s.ExecutionsID,
|
||||
Kubeconfig: encodedKubeconfig,
|
||||
SourcePeerID: localPeerID,
|
||||
DestPeerID: destPeerID,
|
||||
OriginID: originID,
|
||||
}
|
||||
if destPeerID == localPeerID {
|
||||
s.InitializeAsTarget(ctx, kube)
|
||||
return
|
||||
}
|
||||
// Publish the kubeconfig on NATS so the target peer can proceed
|
||||
payload, err := json.Marshal(kube)
|
||||
if err != nil {
|
||||
logger.Error().Msg("InitializeAsSource: failed to marshal kubeconfig event: " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if b, err := json.Marshal(&tools.PropalgationMessage{
|
||||
DataType: -1,
|
||||
Action: tools.PB_ADMIRALTY_CONFIG,
|
||||
Payload: payload,
|
||||
}); err == nil {
|
||||
go tools.NewNATSCaller().SetNATSPub(tools.PROPALGATION_EVENT, tools.NATSResponse{
|
||||
FromApp: "oc-datacenter",
|
||||
Datatype: -1,
|
||||
User: "",
|
||||
Method: int(tools.PROPALGATION_EVENT),
|
||||
Payload: b,
|
||||
})
|
||||
}
|
||||
logger.Info().Msg("InitializeAsSource: kubeconfig published for ns-" + s.ExecutionsID)
|
||||
}
|
||||
|
||||
// InitializeAsTarget is called on the peer that acts as the TARGET cluster (scheduler).
|
||||
// It waits for the kubeconfig published by the source peer via NATS, then creates
|
||||
// the Secret, AdmiraltyTarget, and polls until the virtual node appears.
|
||||
// kubeconfigCh must be obtained from RegisterKubeconfigWaiter before this goroutine starts.
|
||||
func (s *AdmiraltySetter) InitializeAsTarget(ctx context.Context, kubeconfigObj KubeconfigEvent) {
|
||||
logger := oclib.GetLogger()
|
||||
defer kubeconfigChannels.Delete(s.ExecutionsID)
|
||||
|
||||
logger.Info().Msg("InitializeAsTarget: waiting for kubeconfig from source peer ns-" + s.ExecutionsID)
|
||||
kubeconfigData := kubeconfigObj.Kubeconfig
|
||||
|
||||
serv, err := tools.NewKubernetesService(conf.GetConfig().KubeHost+":"+conf.GetConfig().KubePort,
|
||||
conf.GetConfig().KubeCA, conf.GetConfig().KubeCert, conf.GetConfig().KubeData)
|
||||
if err != nil {
|
||||
logger.Error().Msg("InitializeAsTarget: failed to create service: " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// 1. Create the namespace
|
||||
logger.Info().Msg("InitializeAsTarget: creating Namespace " + s.ExecutionsID)
|
||||
if err := serv.CreateNamespace(ctx, s.ExecutionsID); err != nil && !apierrors.IsAlreadyExists(err) {
|
||||
logger.Error().Msg("InitializeAsTarget: failed to create namespace: " + err.Error())
|
||||
emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err)
|
||||
return
|
||||
}
|
||||
|
||||
// 2. Create the ServiceAccount sa-{executionID}
|
||||
logger.Info().Msg("InitializeAsTarget: creating ServiceAccount sa-" + s.ExecutionsID)
|
||||
if err := serv.CreateServiceAccount(ctx, s.ExecutionsID); err != nil && !apierrors.IsAlreadyExists(err) {
|
||||
logger.Error().Msg("InitializeAsTarget: failed to create service account: " + err.Error())
|
||||
emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err)
|
||||
return
|
||||
}
|
||||
|
||||
// 3. Create the Role
|
||||
roleName := "role-" + s.ExecutionsID
|
||||
logger.Info().Msg("InitializeAsTarget: creating Role " + roleName)
|
||||
if err := serv.CreateRole(ctx, s.ExecutionsID, roleName,
|
||||
[][]string{
|
||||
{"coordination.k8s.io"},
|
||||
{""},
|
||||
{""}},
|
||||
[][]string{
|
||||
{"leases"},
|
||||
{"secrets"},
|
||||
{"pods"}},
|
||||
[][]string{
|
||||
{"get", "create", "update"},
|
||||
{"get"},
|
||||
{"patch"}},
|
||||
); err != nil && !apierrors.IsAlreadyExists(err) {
|
||||
logger.Error().Msg("InitializeAsTarget: failed to create role: " + err.Error())
|
||||
emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err)
|
||||
return
|
||||
}
|
||||
|
||||
// 4. Create the RoleBinding
|
||||
rbName := "rb-" + s.ExecutionsID
|
||||
logger.Info().Msg("InitializeAsTarget: creating RoleBinding " + rbName)
|
||||
if err := serv.CreateRoleBinding(ctx, s.ExecutionsID, rbName, roleName); err != nil && !apierrors.IsAlreadyExists(err) {
|
||||
logger.Error().Msg("InitializeAsTarget: failed to create role binding: " + err.Error())
|
||||
emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Create the Secret from the source peer's kubeconfig (inlined from CreateKubeSecret controller)
|
||||
logger.Info().Msg("InitializeAsTarget: creating Secret ns-" + s.ExecutionsID)
|
||||
if _, err := serv.CreateKubeconfigSecret(ctx, kubeconfigData, s.ExecutionsID, kubeconfigObj.SourcePeerID); err != nil {
|
||||
logger.Error().Msg("InitializeAsTarget: failed to create kubeconfig secret: " + err.Error())
|
||||
emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Create the AdmiraltyTarget resource (inlined from CreateAdmiraltyTarget controller)
|
||||
logger.Info().Msg("InitializeAsTarget: creating AdmiraltyTarget ns-" + s.ExecutionsID)
|
||||
resp, err := serv.CreateAdmiraltyTarget(ctx, s.ExecutionsID, kubeconfigObj.SourcePeerID)
|
||||
if err != nil || resp == nil {
|
||||
logger.Error().Msg(fmt.Sprintf("InitializeAsTarget: failed to create admiralty target: %v", err))
|
||||
if err == nil {
|
||||
err = fmt.Errorf("CreateAdmiraltyTarget returned nil response")
|
||||
}
|
||||
emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Poll until the virtual node appears (inlined from GetNodeReady controller)
|
||||
logger.Info().Msg("InitializeAsTarget: waiting for virtual node ns-" + s.ExecutionsID)
|
||||
s.waitForNode(ctx, serv, kubeconfigObj.SourcePeerID)
|
||||
emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, kubeconfigData, nil)
|
||||
}
|
||||
|
||||
// waitForNode polls GetOneNode until the Admiralty virtual node appears on this cluster.
|
||||
func (s *AdmiraltySetter) waitForNode(ctx context.Context, serv *tools.KubernetesService, sourcePeerID string) {
|
||||
logger := oclib.GetLogger()
|
||||
for i := range 5 {
|
||||
time.Sleep(10 * time.Second)
|
||||
node, err := serv.GetOneNode(ctx, s.ExecutionsID, sourcePeerID)
|
||||
if err == nil && node != nil {
|
||||
s.NodeName = node.Name
|
||||
logger.Info().Msg("waitForNode: node ready: " + s.NodeName)
|
||||
return
|
||||
}
|
||||
if i == 4 {
|
||||
logger.Error().Msg("waitForNode: node never appeared for ns-" + s.ExecutionsID)
|
||||
return
|
||||
}
|
||||
logger.Info().Msg("waitForNode: node not ready yet, retrying...")
|
||||
}
|
||||
}
|
||||
|
||||
// TeardownAsTarget destroys all Admiralty resources created by InitializeAsTarget on the
|
||||
// target (scheduler) cluster: the AdmiraltyTarget CRD, the ServiceAccount, the Role,
|
||||
// the RoleBinding, and the namespace (namespace deletion cascades the rest).
|
||||
func (s *AdmiraltySetter) TeardownAsTarget(ctx context.Context, originID string) {
|
||||
logger := oclib.GetLogger()
|
||||
serv, err := tools.NewKubernetesService(conf.GetConfig().KubeHost+":"+conf.GetConfig().KubePort,
|
||||
conf.GetConfig().KubeCA, conf.GetConfig().KubeCert, conf.GetConfig().KubeData)
|
||||
if err != nil {
|
||||
logger.Error().Msg("TeardownAsTarget: failed to create k8s service: " + err.Error())
|
||||
return
|
||||
}
|
||||
if err := serv.DeleteNamespace(ctx, s.ExecutionsID, func() {
|
||||
logger.Info().Msg("TeardownAsTarget: namespace " + s.ExecutionsID + " deleted")
|
||||
}); err != nil {
|
||||
logger.Error().Msg("TeardownAsTarget: " + err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// TeardownAsSource destroys all Admiralty resources created by InitializeAsSource on the
|
||||
// source (compute) cluster: the AdmiraltySource CRD, the ServiceAccount, and the namespace.
|
||||
// The namespace deletion cascades the Role and RoleBinding.
|
||||
func (s *AdmiraltySetter) TeardownAsSource(ctx context.Context) {
|
||||
logger := oclib.GetLogger()
|
||||
host := conf.GetConfig().KubeHost + ":" + conf.GetConfig().KubePort
|
||||
ca := conf.GetConfig().KubeCA
|
||||
cert := conf.GetConfig().KubeCert
|
||||
data := conf.GetConfig().KubeData
|
||||
|
||||
// Delete the AdmiraltySource CRD via dynamic client
|
||||
gvrSources := schema.GroupVersionResource{
|
||||
Group: "multicluster.admiralty.io", Version: "v1alpha1", Resource: "sources",
|
||||
}
|
||||
if dyn, err := tools.NewDynamicClient(host, ca, cert, data); err != nil {
|
||||
logger.Error().Msg("TeardownAsSource: failed to create dynamic client: " + err.Error())
|
||||
} else if err := dyn.Resource(gvrSources).Namespace(s.ExecutionsID).Delete(
|
||||
ctx, "source-"+s.ExecutionsID, metav1.DeleteOptions{},
|
||||
); err != nil {
|
||||
logger.Error().Msg("TeardownAsSource: failed to delete AdmiraltySource: " + err.Error())
|
||||
}
|
||||
|
||||
// Delete the namespace (cascades SA, Role, RoleBinding)
|
||||
serv, err := tools.NewKubernetesService(host, ca, cert, data)
|
||||
if err != nil {
|
||||
logger.Error().Msg("TeardownAsSource: failed to create k8s service: " + err.Error())
|
||||
return
|
||||
}
|
||||
if err := serv.Set.CoreV1().Namespaces().Delete(ctx, s.ExecutionsID, metav1.DeleteOptions{}); err != nil {
|
||||
logger.Error().Msg("TeardownAsSource: failed to delete namespace: " + err.Error())
|
||||
return
|
||||
}
|
||||
logger.Info().Msg("TeardownAsSource: namespace " + s.ExecutionsID + " deleted")
|
||||
}
|
||||
|
||||
// buildHostKubeWithToken builds a kubeconfig pointing to this peer's cluster,
|
||||
// authenticated with the provided service-account token.
|
||||
func buildHostKubeWithToken(token string) (*models.KubeConfigValue, error) {
|
||||
if len(token) == 0 {
|
||||
return nil, fmt.Errorf("buildHostKubeWithToken: empty token")
|
||||
}
|
||||
encodedCA := base64.StdEncoding.EncodeToString([]byte(conf.GetConfig().KubeCA))
|
||||
return &models.KubeConfigValue{
|
||||
APIVersion: "v1",
|
||||
CurrentContext: "default",
|
||||
Kind: "Config",
|
||||
Preferences: struct{}{},
|
||||
Clusters: []models.KubeconfigNamedCluster{{
|
||||
Name: "default",
|
||||
Cluster: models.KubeconfigCluster{
|
||||
Server: "https://" + conf.GetConfig().KubeHost + ":6443",
|
||||
CertificateAuthorityData: encodedCA,
|
||||
},
|
||||
}},
|
||||
Contexts: []models.KubeconfigNamedContext{{
|
||||
Name: "default",
|
||||
Context: models.KubeconfigContext{Cluster: "default", User: "default"},
|
||||
}},
|
||||
Users: []models.KubeconfigUser{{
|
||||
Name: "default",
|
||||
User: models.KubeconfigUserKeyPair{Token: token},
|
||||
}},
|
||||
}, nil
|
||||
}
|
||||
Reference in New Issue
Block a user