package infrastructure import ( "context" "encoding/base64" "encoding/json" "fmt" "sync" "time" "oc-datacenter/conf" "oc-datacenter/infrastructure/monitor" "oc-datacenter/models" oclib "cloud.o-forge.io/core/oc-lib" "cloud.o-forge.io/core/oc-lib/tools" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" ) // kubeconfigChannels holds channels waiting for kubeconfig delivery (keyed by executionID). var kubeconfigChannels sync.Map // kubeconfigEvent is the NATS payload used to transfer the kubeconfig from the source peer to the target peer. type KubeconfigEvent struct { DestPeerID string `json:"dest_peer_id"` ExecutionsID string `json:"executions_id"` Kubeconfig string `json:"kubeconfig"` SourcePeerID string `json:"source_peer_id"` // OriginID is the peer that initiated the provisioning request. // The PB_CONSIDERS response is routed back to this peer. OriginID string `json:"origin_id"` } // admiraltyConsidersPayload is the PB_CONSIDERS payload emitted after admiralty provisioning. type admiraltyConsidersPayload struct { OriginID string `json:"origin_id"` ExecutionsID string `json:"executions_id"` Secret string `json:"secret,omitempty"` Error *string `json:"error,omitempty"` } // emitAdmiraltyConsiders publishes a PB_CONSIDERS back to OriginID with the result // of the admiralty provisioning. secret is the base64-encoded kubeconfig; err is nil on success. func emitAdmiraltyConsiders(executionsID, originID, secret string, provErr error) { var errStr *string if provErr != nil { s := provErr.Error() errStr = &s } payload, _ := json.Marshal(admiraltyConsidersPayload{ OriginID: originID, ExecutionsID: executionsID, Secret: secret, Error: errStr, }) b, _ := json.Marshal(&tools.PropalgationMessage{ DataType: tools.COMPUTE_RESOURCE.EnumIndex(), Action: tools.PB_CONSIDERS, Payload: payload, }) go tools.NewNATSCaller().SetNATSPub(tools.PROPALGATION_EVENT, tools.NATSResponse{ FromApp: "oc-datacenter", Datatype: -1, Method: int(tools.PROPALGATION_EVENT), Payload: b, }) } // AdmiraltySetter carries the execution context for an admiralty pairing. type AdmiraltySetter struct { ExecutionsID string // execution ID, used as the Kubernetes namespace NodeName string // name of the virtual node created by Admiralty on the target cluster } func NewAdmiraltySetter(execIDS string) *AdmiraltySetter { return &AdmiraltySetter{ ExecutionsID: execIDS, } } // InitializeAsSource is called on the peer that acts as the SOURCE cluster (compute provider). // It creates the AdmiraltySource resource, generates a kubeconfig for the target peer, // and publishes it on NATS so the target peer can complete its side of the setup. func (s *AdmiraltySetter) InitializeAsSource(ctx context.Context, localPeerID string, destPeerID string, originID string) { logger := oclib.GetLogger() serv, err := tools.NewKubernetesService(conf.GetConfig().KubeHost+":"+conf.GetConfig().KubePort, conf.GetConfig().KubeCA, conf.GetConfig().KubeCert, conf.GetConfig().KubeData) if err != nil { logger.Error().Msg("InitializeAsSource: failed to create service: " + err.Error()) return } // Create the AdmiraltySource resource on this cluster (inlined from CreateAdmiraltySource controller) logger.Info().Msg("Creating AdmiraltySource ns-" + s.ExecutionsID) _, err = serv.CreateAdmiraltySource(ctx, s.ExecutionsID) if err != nil && !apierrors.IsAlreadyExists(err) { logger.Error().Msg("InitializeAsSource: failed to create source: " + err.Error()) return } // Generate a service-account token for the namespace (inlined from GetAdmiraltyKubeconfig controller) token, err := serv.GenerateToken(ctx, s.ExecutionsID, 3600) if err != nil { logger.Error().Msg("InitializeAsSource: failed to generate token for ns-" + s.ExecutionsID + ": " + err.Error()) return } kubeconfig, err := buildHostKubeWithToken(token) if err != nil { logger.Error().Msg("InitializeAsSource: " + err.Error()) return } b, err := json.Marshal(kubeconfig) if err != nil { logger.Error().Msg("InitializeAsSource: failed to marshal kubeconfig: " + err.Error()) return } encodedKubeconfig := base64.StdEncoding.EncodeToString(b) kube := KubeconfigEvent{ ExecutionsID: s.ExecutionsID, Kubeconfig: encodedKubeconfig, SourcePeerID: localPeerID, DestPeerID: destPeerID, OriginID: originID, } if destPeerID == localPeerID { s.InitializeAsTarget(ctx, kube) return } // Publish the kubeconfig on NATS so the target peer can proceed payload, err := json.Marshal(kube) if err != nil { logger.Error().Msg("InitializeAsSource: failed to marshal kubeconfig event: " + err.Error()) return } if b, err := json.Marshal(&tools.PropalgationMessage{ DataType: -1, Action: tools.PB_ADMIRALTY_CONFIG, Payload: payload, }); err == nil { go tools.NewNATSCaller().SetNATSPub(tools.PROPALGATION_EVENT, tools.NATSResponse{ FromApp: "oc-datacenter", Datatype: -1, User: "", Method: int(tools.PROPALGATION_EVENT), Payload: b, }) } logger.Info().Msg("InitializeAsSource: kubeconfig published for ns-" + s.ExecutionsID) } // InitializeAsTarget is called on the peer that acts as the TARGET cluster (scheduler). // It waits for the kubeconfig published by the source peer via NATS, then creates // the Secret, AdmiraltyTarget, and polls until the virtual node appears. // kubeconfigCh must be obtained from RegisterKubeconfigWaiter before this goroutine starts. func (s *AdmiraltySetter) InitializeAsTarget(ctx context.Context, kubeconfigObj KubeconfigEvent) { logger := oclib.GetLogger() defer kubeconfigChannels.Delete(s.ExecutionsID) logger.Info().Msg("InitializeAsTarget: waiting for kubeconfig from source peer ns-" + s.ExecutionsID) kubeconfigData := kubeconfigObj.Kubeconfig serv, err := tools.NewKubernetesService(conf.GetConfig().KubeHost+":"+conf.GetConfig().KubePort, conf.GetConfig().KubeCA, conf.GetConfig().KubeCert, conf.GetConfig().KubeData) if err != nil { logger.Error().Msg("InitializeAsTarget: failed to create service: " + err.Error()) return } // 1. Create the namespace logger.Info().Msg("InitializeAsTarget: creating Namespace " + s.ExecutionsID) if err := serv.CreateNamespace(ctx, s.ExecutionsID); err != nil && !apierrors.IsAlreadyExists(err) { logger.Error().Msg("InitializeAsTarget: failed to create namespace: " + err.Error()) emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err) return } // 2. Create the ServiceAccount sa-{executionID} logger.Info().Msg("InitializeAsTarget: creating ServiceAccount sa-" + s.ExecutionsID) if err := serv.CreateServiceAccount(ctx, s.ExecutionsID); err != nil && !apierrors.IsAlreadyExists(err) { logger.Error().Msg("InitializeAsTarget: failed to create service account: " + err.Error()) emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err) return } // 3. Create the Role roleName := "role-" + s.ExecutionsID logger.Info().Msg("InitializeAsTarget: creating Role " + roleName) if err := serv.CreateRole(ctx, s.ExecutionsID, roleName, [][]string{ {"coordination.k8s.io"}, {""}, {""}}, [][]string{ {"leases"}, {"secrets"}, {"pods"}}, [][]string{ {"get", "create", "update"}, {"get"}, {"patch"}}, ); err != nil && !apierrors.IsAlreadyExists(err) { logger.Error().Msg("InitializeAsTarget: failed to create role: " + err.Error()) emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err) return } // 4. Create the RoleBinding rbName := "rb-" + s.ExecutionsID logger.Info().Msg("InitializeAsTarget: creating RoleBinding " + rbName) if err := serv.CreateRoleBinding(ctx, s.ExecutionsID, rbName, roleName); err != nil && !apierrors.IsAlreadyExists(err) { logger.Error().Msg("InitializeAsTarget: failed to create role binding: " + err.Error()) emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err) return } // Create the Secret from the source peer's kubeconfig (inlined from CreateKubeSecret controller) logger.Info().Msg("InitializeAsTarget: creating Secret ns-" + s.ExecutionsID) if _, err := serv.CreateKubeconfigSecret(ctx, kubeconfigData, s.ExecutionsID, kubeconfigObj.SourcePeerID); err != nil { logger.Error().Msg("InitializeAsTarget: failed to create kubeconfig secret: " + err.Error()) emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err) return } // Create the AdmiraltyTarget resource (inlined from CreateAdmiraltyTarget controller) logger.Info().Msg("InitializeAsTarget: creating AdmiraltyTarget ns-" + s.ExecutionsID) resp, err := serv.CreateAdmiraltyTarget(ctx, s.ExecutionsID, kubeconfigObj.SourcePeerID) if err != nil || resp == nil { logger.Error().Msg(fmt.Sprintf("InitializeAsTarget: failed to create admiralty target: %v", err)) if err == nil { err = fmt.Errorf("CreateAdmiraltyTarget returned nil response") } emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err) return } // Poll until the virtual node appears (inlined from GetNodeReady controller) logger.Info().Msg("InitializeAsTarget: waiting for virtual node ns-" + s.ExecutionsID) s.waitForNode(ctx, serv, kubeconfigObj.SourcePeerID) emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, kubeconfigData, nil) } // waitForNode polls GetOneNode until the Admiralty virtual node appears on this cluster. func (s *AdmiraltySetter) waitForNode(ctx context.Context, serv *tools.KubernetesService, sourcePeerID string) { logger := oclib.GetLogger() for i := range 5 { time.Sleep(10 * time.Second) node, err := serv.GetOneNode(ctx, s.ExecutionsID, sourcePeerID) if err == nil && node != nil { s.NodeName = node.Name logger.Info().Msg("waitForNode: node ready: " + s.NodeName) return } if i == 4 { logger.Error().Msg("waitForNode: node never appeared for ns-" + s.ExecutionsID) return } logger.Info().Msg("waitForNode: node not ready yet, retrying...") } } // TeardownAsTarget destroys all Admiralty resources created by InitializeAsTarget on the // target (scheduler) cluster: the AdmiraltyTarget CRD, the ServiceAccount, the Role, // the RoleBinding, and the namespace (namespace deletion cascades the rest). func (s *AdmiraltySetter) TeardownAsTarget(ctx context.Context, originID string) { logger := oclib.GetLogger() serv, err := tools.NewKubernetesService(conf.GetConfig().KubeHost+":"+conf.GetConfig().KubePort, conf.GetConfig().KubeCA, conf.GetConfig().KubeCert, conf.GetConfig().KubeData) if err != nil { logger.Error().Msg("TeardownAsTarget: failed to create k8s service: " + err.Error()) return } if err := serv.DeleteNamespace(ctx, s.ExecutionsID, func() { logger.Info().Msg("TeardownAsTarget: namespace " + s.ExecutionsID + " deleted") defer monitor.StreamRegistry.Register(s.ExecutionsID) }); err != nil { logger.Error().Msg("TeardownAsTarget: " + err.Error()) return } } // TeardownAsSource destroys all Admiralty resources created by InitializeAsSource on the // source (compute) cluster: the AdmiraltySource CRD, the ServiceAccount, and the namespace. // The namespace deletion cascades the Role and RoleBinding. func (s *AdmiraltySetter) TeardownAsSource(ctx context.Context) { logger := oclib.GetLogger() host := conf.GetConfig().KubeHost + ":" + conf.GetConfig().KubePort ca := conf.GetConfig().KubeCA cert := conf.GetConfig().KubeCert data := conf.GetConfig().KubeData // Delete the AdmiraltySource CRD via dynamic client gvrSources := schema.GroupVersionResource{ Group: "multicluster.admiralty.io", Version: "v1alpha1", Resource: "sources", } if dyn, err := tools.NewDynamicClient(host, ca, cert, data); err != nil { logger.Error().Msg("TeardownAsSource: failed to create dynamic client: " + err.Error()) } else if err := dyn.Resource(gvrSources).Namespace(s.ExecutionsID).Delete( ctx, "source-"+s.ExecutionsID, metav1.DeleteOptions{}, ); err != nil { logger.Error().Msg("TeardownAsSource: failed to delete AdmiraltySource: " + err.Error()) } // Delete the namespace (cascades SA, Role, RoleBinding) serv, err := tools.NewKubernetesService(host, ca, cert, data) if err != nil { logger.Error().Msg("TeardownAsSource: failed to create k8s service: " + err.Error()) return } if err := serv.Set.CoreV1().Namespaces().Delete(ctx, s.ExecutionsID, metav1.DeleteOptions{}); err != nil { logger.Error().Msg("TeardownAsSource: failed to delete namespace: " + err.Error()) return } logger.Info().Msg("TeardownAsSource: namespace " + s.ExecutionsID + " deleted") } // buildHostKubeWithToken builds a kubeconfig pointing to this peer's cluster, // authenticated with the provided service-account token. func buildHostKubeWithToken(token string) (*models.KubeConfigValue, error) { if len(token) == 0 { return nil, fmt.Errorf("buildHostKubeWithToken: empty token") } encodedCA := base64.StdEncoding.EncodeToString([]byte(conf.GetConfig().KubeCA)) return &models.KubeConfigValue{ APIVersion: "v1", CurrentContext: "default", Kind: "Config", Preferences: struct{}{}, Clusters: []models.KubeconfigNamedCluster{{ Name: "default", Cluster: models.KubeconfigCluster{ Server: "https://" + conf.GetConfig().KubeHost + ":6443", CertificateAuthorityData: encodedCA, }, }}, Contexts: []models.KubeconfigNamedContext{{ Name: "default", Context: models.KubeconfigContext{Cluster: "default", User: "default"}, }}, Users: []models.KubeconfigUser{{ Name: "default", User: models.KubeconfigUserKeyPair{Token: token}, }}, }, nil }