diff --git a/cookbook/README.md b/cookbook/README.md index 3416522..94ddb24 100644 --- a/cookbook/README.md +++ b/cookbook/README.md @@ -78,6 +78,7 @@ cookbook/ │ └── deploy_nginx.yaml # Nginx YAML 配置 └── supernode/ # 超级节点 ├── README.md + ├── delete_supernode.py # 删除超级节点 ├── deploy_gpu_pod.py # 部署 GPU Pod └── gpu_pod_examples.yaml # GPU Pod 示例配置 ``` @@ -142,6 +143,7 @@ python3 supernode/deploy_gpu_pod.py \ | 脚本 | 语言 | 功能 | 文档链接 | |------|------|------|---------| | `deploy_gpu_pod.py` | Python | 在超级节点上部署 GPU Pod | [docs](../docs/basics/supernode/02-create-supernode.md) | +| `delete_supernode.py` | Python | 删除超级节点,默认 dry-run | [docs](../src/content/docs/basics/supernode/03-delete-supernode.md) | --- diff --git a/cookbook/supernode/README.md b/cookbook/supernode/README.md index 508c671..09b14bb 100644 --- a/cookbook/supernode/README.md +++ b/cookbook/supernode/README.md @@ -4,7 +4,7 @@ ## 📖 简介 -本目录提供在 TKE 超级节点上部署 GPU Pod 的完整示例,包括 Python 脚本和 YAML 配置文件。 +本目录提供 TKE 超级节点相关的完整示例,包括 GPU Pod 部署、超级节点删除脚本和 YAML 配置文件。 **文档链接**: [GPU Pod 最佳实践](https://tke-workshop.github.io/ai-ml/04-gpu-pod-best-practices/) @@ -15,6 +15,7 @@ | 文件 | 类型 | 功能 | 推荐场景 | |------|------|------|---------| | `deploy_gpu_pod.py` | Python | GPU Pod 部署脚本 | 自动化部署、批量操作 | +| `delete_supernode.py` | Python | 删除超级节点,默认 dry-run | 清理闲置超级节点、批量删除前检查 | | `gpu_pod_examples.yaml` | YAML | GPU Pod 配置示例集合 | kubectl 直接部署 | --- @@ -135,6 +136,29 @@ python3 deploy_gpu_pod.py --delete --name gpu-inference python3 deploy_gpu_pod.py --help ``` +### 删除超级节点 + +```bash +# 预检查删除计划,不提交删除 +python3 delete_supernode.py \ + --cluster-id cls-xxxxxxxx \ + --node-name eklet-subnet-xxxxxxxx-0 + +# 节点已无 Pod 后提交安全删除 +python3 delete_supernode.py \ + --cluster-id cls-xxxxxxxx \ + --node-name eklet-subnet-xxxxxxxx-0 \ + --confirm-delete \ + --wait + +# 已确认业务风险后强制删除 +python3 delete_supernode.py \ + --cluster-id cls-xxxxxxxx \ + --node-name eklet-subnet-xxxxxxxx-0 \ + --force \ + --confirm-delete +``` + ### 方式二:使用 YAML 配置 #### 1. 部署单个示例 diff --git a/cookbook/supernode/delete_supernode.py b/cookbook/supernode/delete_supernode.py new file mode 100644 index 0000000..7251f7b --- /dev/null +++ b/cookbook/supernode/delete_supernode.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +""" +删除 TKE 超级节点示例脚本 + +功能: 删除指定 TKE 超级节点,默认 dry-run,需显式确认后才会提交删除请求 +使用方法: python3 delete_supernode.py --cluster-id cls-xxxxxxxx --node-name eklet-subnet-xxxxxxxx-0 +文档链接: https://tke-workshop.github.io/basics/supernode/03-delete-supernode/ +""" + +from __future__ import annotations + +import argparse +import json +import logging +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", +) +logger = logging.getLogger(__name__) + + +class LogContext: + def __init__(self, logger: logging.Logger, operation: str): + self.logger = logger + self.operation = operation + + def __enter__(self): + self.logger.info("开始: %s", self.operation) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type is None: + self.logger.info("完成: %s", self.operation) + else: + self.logger.error("失败: %s - %s", self.operation, exc_val) + return False + + +def run_kubectl(args: list[str], kubeconfig: str | None = None) -> subprocess.CompletedProcess: + cmd = ["kubectl"] + if kubeconfig: + cmd.extend(["--kubeconfig", kubeconfig]) + cmd.extend(args) + + return subprocess.run(cmd, capture_output=True, text=True, check=False) + + +def get_node_pods(node_name: str, kubeconfig: str | None = None) -> list[str]: + result = run_kubectl( + [ + "get", + "pods", + "--all-namespaces", + "--field-selector", + f"spec.nodeName={node_name}", + "-o", + "jsonpath={range .items[*]}{.metadata.namespace}/{.metadata.name}{\"\\n\"}{end}", + ], + kubeconfig=kubeconfig, + ) + + if result.returncode != 0: + raise RuntimeError(f"kubectl 查询节点 Pod 失败: {result.stderr.strip() or result.stdout.strip()}") + + return [line for line in result.stdout.splitlines() if line.strip()] + + +def build_delete_request( + cluster_id: str, + node_names: list[str], + force: bool, +) -> dict[str, Any]: + return { + "ClusterId": cluster_id, + "NodeNames": node_names, + "Force": force, + } + + +def build_sdk_delete_request(cluster_id: str, node_names: list[str], force: bool): + from tencentcloud.tke.v20180525 import models + + req = models.DeleteClusterVirtualNodeRequest() + req.ClusterId = cluster_id + req.NodeNames = node_names + req.Force = force + return req + + +def delete_supernodes( + cluster_id: str, + region: str, + node_names: list[str], + force: bool = False, + confirm_delete: bool = False, + skip_pod_check: bool = False, + kubeconfig: str | None = None, +): + """ + 删除 TKE 超级节点。默认只打印计划,必须传入 confirm_delete=True 才会提交删除请求。 + """ + with LogContext(logger, f"删除超级节点: {', '.join(node_names)}"): + logger.info(f"目标集群: {cluster_id}") + logger.info(f"地域: {region}") + logger.info(f"强制删除: {force}") + + if not skip_pod_check: + for node_name in node_names: + pods = get_node_pods(node_name, kubeconfig=kubeconfig) + if pods: + logger.warning(f"节点 {node_name} 上仍有 Pod:") + for pod in pods: + logger.warning(f" - {pod}") + if not force: + raise RuntimeError( + "检测到运行中的 Pod。请先执行 kubectl drain,或在已确认业务风险后传入 --force。" + ) + else: + logger.info(f"节点 {node_name} 上未发现 Pod") + else: + logger.warning("已跳过 kubectl Pod 检查,请确保删除前已确认业务影响") + + request_payload = build_delete_request(cluster_id, node_names, force) + logger.info("DeleteClusterVirtualNode 请求参数:") + logger.info(json.dumps(request_payload, ensure_ascii=False, indent=2)) + + if not confirm_delete: + logger.warning("当前为 dry-run,未提交 DeleteClusterVirtualNode 请求。") + logger.warning("确认节点无业务负载或已接受强制删除风险后,重新执行并添加 --confirm-delete。") + return None + + from common.auth import get_tke_client + from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException + + client = get_tke_client(region) + req = build_sdk_delete_request(cluster_id, node_names, force) + + try: + resp = client.DeleteClusterVirtualNode(req) + except TencentCloudSDKException as e: + logger.error(f"删除超级节点失败: {e}") + logger.error(f"错误码: {e.code}") + logger.error(f"错误消息: {e.message}") + raise + + logger.info("删除超级节点请求已提交") + logger.info(f"RequestId: {resp.RequestId}") + return resp + + +def wait_nodes_deleted( + node_names: list[str], + kubeconfig: str | None = None, + timeout: int = 600, +): + start_time = time.time() + remaining = set(node_names) + + logger.info(f"等待超级节点从 Kubernetes 节点列表中移除: {', '.join(node_names)}") + while time.time() - start_time < timeout: + for node_name in list(remaining): + result = run_kubectl(["get", "node", node_name, "-o", "name"], kubeconfig=kubeconfig) + if result.returncode != 0: + remaining.discard(node_name) + logger.info(f"节点已移除: {node_name}") + + if not remaining: + return + + logger.info(f"仍在等待节点移除: {', '.join(sorted(remaining))}") + time.sleep(10) + + raise TimeoutError(f"等待超级节点删除超时: {', '.join(sorted(remaining))}") + + +def main(): + parser = argparse.ArgumentParser( + description="删除 TKE 超级节点,默认 dry-run,需显式确认后才会提交删除请求", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +示例: + # 预检查删除计划,不提交删除 + python3 delete_supernode.py \\ + --cluster-id cls-xxxxxxxx \\ + --node-name eklet-subnet-xxxxxxxx-0 + + # 节点已无 Pod 后提交安全删除 + python3 delete_supernode.py \\ + --cluster-id cls-xxxxxxxx \\ + --node-name eklet-subnet-xxxxxxxx-0 \\ + --confirm-delete \\ + --wait + + # 已确认业务风险后强制删除 + python3 delete_supernode.py \\ + --cluster-id cls-xxxxxxxx \\ + --node-name eklet-subnet-xxxxxxxx-0 \\ + --force \\ + --confirm-delete + """, + ) + + parser.add_argument("--cluster-id", required=True, help="要操作的 TKE 集群 ID") + parser.add_argument("--region", default="ap-guangzhou", help="地域") + parser.add_argument( + "--node-name", + action="append", + dest="node_names", + required=True, + help="超级节点名称,可重复传入多个", + ) + parser.add_argument( + "--force", + action="store_true", + help="强制删除;若节点上有运行中的 Pod,API 仍会提交删除请求", + ) + parser.add_argument( + "--confirm-delete", + action="store_true", + help="确认已完成业务风险检查,并提交 DeleteClusterVirtualNode 请求", + ) + parser.add_argument( + "--skip-pod-check", + action="store_true", + help="跳过本地 kubectl Pod 检查;仅在无法访问 kube-apiserver 且已人工确认时使用", + ) + parser.add_argument("--kubeconfig", help="kubeconfig 文件路径,留空使用默认配置") + parser.add_argument("--wait", action="store_true", help="提交删除后等待节点从 Kubernetes 中移除") + parser.add_argument("--timeout", type=int, default=600, help="等待删除完成的超时时间(秒)") + + args = parser.parse_args() + + try: + resp = delete_supernodes( + cluster_id=args.cluster_id, + region=args.region, + node_names=args.node_names, + force=args.force, + confirm_delete=args.confirm_delete, + skip_pod_check=args.skip_pod_check, + kubeconfig=args.kubeconfig, + ) + + if resp and args.wait: + wait_nodes_deleted(args.node_names, kubeconfig=args.kubeconfig, timeout=args.timeout) + except Exception as e: + logger.error(f"删除超级节点失败: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/src/content/docs/basics/supernode/03-delete-supernode.md b/src/content/docs/basics/supernode/03-delete-supernode.md index f2f088e..186837b 100644 --- a/src/content/docs/basics/supernode/03-delete-supernode.md +++ b/src/content/docs/basics/supernode/03-delete-supernode.md @@ -461,7 +461,28 @@ delete_virtual_node_safely("cls-xxxxxxxx", "eklet-subnet-xxxxxxxx-0") ## Cookbook 示例 -完整可执行代码示例: TKE 超级节点删除 Cookbook +完整可执行代码示例: TKE 超级节点删除 Cookbook;源码见 [`cookbook/supernode/delete_supernode.py`](https://github.com/tke-workshop/tke-workshop.github.io/blob/main/cookbook/supernode/delete_supernode.py)。 + +```bash +# 预检查删除计划,不提交删除 +python3 cookbook/supernode/delete_supernode.py \ + --cluster-id cls-xxxxxxxx \ + --node-name eklet-subnet-xxxxxxxx-0 + +# 节点已无 Pod 后提交安全删除 +python3 cookbook/supernode/delete_supernode.py \ + --cluster-id cls-xxxxxxxx \ + --node-name eklet-subnet-xxxxxxxx-0 \ + --confirm-delete \ + --wait + +# 已确认业务风险后强制删除 +python3 cookbook/supernode/delete_supernode.py \ + --cluster-id cls-xxxxxxxx \ + --node-name eklet-subnet-xxxxxxxx-0 \ + --force \ + --confirm-delete +``` ---