# 完整代码（增加测试集和B中心保存）
import pandas as pd
import re
import numpy as np
from sklearn.model_selection import train_test_split
from imblearn.over_sampling import SMOTENC
import os

# 自定义路径处理
def safe_path(path):
    return os.path.normpath(path)

# 文件路径配置（新增测试集和B中心路径）
PATHS = {
    'feature': safe_path(r'H:/taskPONV/stable/stable_features_data.csv'),
    'label': safe_path(r'H:/taskPONV/stable/label320_96.csv'),
    'output': {
        'resampled': safe_path(r'H:/taskPONV/stable/resampled_train.csv'),
        'test_set': safe_path(r'H:/taskPONV/stable/original_test.csv'),
        'b_center': safe_path(r'H:/taskPONV/stable/B_center_data.csv')
    }
}

# ID处理函数保持不变
def process_id(raw_id):
    clean_id = str(raw_id).lower().strip()
    match = re.match(r'(sub\d{3,4})(\.nii\.gz)?', clean_id)
    return match.group(1) if match else clean_id.split('.')[0]

# 数据加载函数保持不变
def load_data():
    features = pd.read_csv(PATHS['feature'], dtype={'ID': 'string'})
    labels = pd.read_csv(PATHS['label'], dtype={'ID': 'string'})
    
    features['ID'] = features['ID'].apply(process_id)
    labels['ID'] = labels['ID'].apply(process_id)
    
    data = pd.merge(features, labels, on='ID', how='inner')
    print(f"\n合并后总样本量: {len(data)}")
    return data

# 主处理流程（新增保存步骤）
def main():
    # 加载数据
    data = load_data()
    
    # 添加中心分类
    data['center'] = data['ID'].apply(
        lambda x: 'A' if re.fullmatch(r'sub\d{3}', x) else 'B'
    )
    print("\n中心分布统计:")
    print(data['center'].value_counts())
    
    # 提取并保存B中心数据
    b_center = data[data['center'] == 'B']
    b_center.to_csv(PATHS['output']['b_center'], index=False)
    print(f"\nB中心数据已保存: {PATHS['output']['b_center']}")
    
    # 处理A中心数据
    a_center = data[data['center'] == 'A']
    print(f"A中心有效样本量: {len(a_center)}")
    
    # 数据划分
    train, test = train_test_split(
        a_center,
        test_size=0.3,
        stratify=a_center['label'],
        random_state=42
    )
    print(f"划分结果：训练集({len(train)}) | 测试集({len(test)})")
    
    # 保存测试集
    test.to_csv(PATHS['output']['test_set'], index=False)
    print(f"测试集已保存: {PATHS['output']['test_set']}")
    
    # 过采样处理（保持不变）
    X_train = train.drop(columns=['label', 'center', 'ID'])
    y_train = train['label']
    
    categorical_features = ['gender', 'smoke']
    cat_indices = [X_train.columns.get_loc(col) for col in categorical_features]
    
    for col in categorical_features:
        X_train[col] = X_train[col].astype('category')
    
    smote_nc = SMOTENC(
        categorical_features=cat_indices,
        sampling_strategy='auto',
        k_neighbors=5,
        random_state=42
    )
    X_res, y_res = smote_nc.fit_resample(X_train, y_train)
    
    resampled_train = pd.DataFrame(X_res, columns=X_train.columns)
    resampled_train['label'] = y_res
    resampled_train = pd.concat([
        resampled_train,
        train[['ID', 'center']].reset_index(drop=True)
    ], axis=1)
    
    # 保存过采样结果
    resampled_train.to_csv(PATHS['output']['resampled'], index=False)
    print(f"\n过采样训练集已保存: {PATHS['output']['resampled']}")

if __name__ == "__main__":
    # 创建输出目录（如果不存在）
    os.makedirs(os.path.dirname(PATHS['output']['resampled']), exist_ok=True)
    main()
import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler
import joblib
import os

# 路径配置
PATHS = {
    'train': r'H:\taskPONV\stable\resampled_train.csv',
    'internal': r'H:\taskPONV\stable\original_test.csv',
    'external': r'H:\taskPONV\stable\B_center_data.csv',
    'scaler': r'H:\taskPONV\stable\std_scaler.pkl',
    'output': {
        'train': r'H:\taskPONV\stable\scaled_train.csv',
        'internal': r'H:\taskPONV\stable\scaled_internal.csv',
        'external': r'H:\taskPONV\stable\scaled_external.csv'
    }
}

# 定义特征类型（根据实际数据调整）
CONTINUOUS_FEATURES = [ 'lh_S_parieto_occipital_curvind', 'rh_Lat_Fis-ant-Horizont_thicknessstd', 'Left-VentralDC_volume',
     'lh_S_calcarine_curvind', 'lh_G_oc-temp_med-Parahip_gauscurv', 'rh_Pole_occipital_curvind',
    'Right-Thalamus_mean_intensity', 'lh_G_oc-temp_med-Parahip_thicknessstd', 'Left-Pallidum_mean_intensity',
    'rh_S_central_foldind', 'rh_G_cuneus_curvind', 'Left-vessel_std', 'rh_Lat_Fis-ant-Vertical_foldind',
    'Left-choroid-plexus_volume', 'lh_G_postcentral_volume', 'rh_S_intrapariet_and_P_trans_foldind',
    'lh_S_postcentral_volume', 'rh_G_parietal_sup_thicknessstd']  # 请替换为实际连续特征列
CATEGORICAL_FEATURES = ['gender', 'smoke']
META_COLUMNS = ['ID', 'center', 'label']

def load_and_validate(path):
    """加载数据集并验证完整性"""
    df = pd.read_csv(path)
    
    # 验证必须存在的列
    required_columns = CONTINUOUS_FEATURES + CATEGORICAL_FEATURES + META_COLUMNS
    missing = set(required_columns) - set(df.columns)
    if missing:
        raise KeyError(f"文件 {os.path.basename(path)} 缺失列: {missing}")
    
    return df

def standardize_dataset():
    """标准化主流程"""
    
    # 加载所有数据集
    train_df = load_and_validate(PATHS['train'])
    internal_df = load_and_validate(PATHS['internal'])
    external_df = load_and_validate(PATHS['external'])
    
    # 初始化标准化器（仅使用训练数据拟合）
    scaler = StandardScaler()
    scaler.fit(train_df[CONTINUOUS_FEATURES])
    
    # 保存标准化器
    joblib.dump(scaler, PATHS['scaler'])
    
    # 定义标准化函数
    def apply_scaling(df, scaler):
        scaled = df.copy()
        scaled[CONTINUOUS_FEATURES] = scaler.transform(df[CONTINUOUS_FEATURES])
        return scaled
    
    # 应用标准化
    scaled_train = apply_scaling(train_df, scaler)
    scaled_internal = apply_scaling(internal_df, scaler)
    scaled_external = apply_scaling(external_df, scaler)
    
    # 保存结果
    scaled_train.to_csv(PATHS['output']['train'], index=False)
    scaled_internal.to_csv(PATHS['output']['internal'], index=False)
    scaled_external.to_csv(PATHS['output']['external'], index=False)
    
    return scaled_train, scaled_internal, scaled_external

# 执行标准化
if __name__ == "__main__":
    # 创建输出目录
    os.makedirs(os.path.dirname(PATHS['output']['train']), exist_ok=True)
    
    # 执行标准化流程
    train_scaled, internal_scaled, external_scaled = standardize_dataset()
    
    # 验证结果
    print("\n标准化验证:")
    print("训练集均值:", np.round(train_scaled[CONTINUOUS_FEATURES].mean(), 4))
    print("内部验证集均值:", np.round(internal_scaled[CONTINUOUS_FEATURES].mean(), 4))
    print("外部验证集均值:", np.round(external_scaled[CONTINUOUS_FEATURES].mean(), 4))
    
    print("\n标准化模型和数据集已保存至:")
    print(f"标准化器: {PATHS['scaler']}")
    print(f"训练集: {PATHS['output']['train']}")
    print(f"内部验证集: {PATHS['output']['internal']}")
    print(f"外部验证集: {PATHS['output']['external']}")
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import StandardScaler
import joblib
import os
import re

# --------------------- 配置参数 ---------------------
PATHS = {
    'train': r'H:\taskPONV\stable\resampled_train.csv',
    'internal': r'H:\taskPONV\stable\original_test.csv',
    'external': r'H:\taskPONV\stable\B_center_data.csv',
    'scaler': r'H:\taskPONV\stable\std_scaler.pkl',
    'output': {
        'train': r'H:\taskPONV\stable\scaled_train.csv',
        'internal': r'H:\taskPONV\stable\scaled_internal.csv',
        'external': r'H:\taskPONV\stable\scaled_external.csv'
    },
    'visualization': r'H:\taskPONV\stable\distribution_plots'  # 可视化输出目录
}

FEATURE_CONFIG = {
    'continuous': [ 'lh_S_parieto_occipital_curvind', 'rh_Lat_Fis-ant-Horizont_thicknessstd', 'Left-VentralDC_volume',
     'lh_S_calcarine_curvind', 'lh_G_oc-temp_med-Parahip_gauscurv', 'rh_Pole_occipital_curvind',
    'Right-Thalamus_mean_intensity', 'lh_G_oc-temp_med-Parahip_thicknessstd', 'Left-Pallidum_mean_intensity',
    'rh_S_central_foldind', 'rh_G_cuneus_curvind', 'Left-vessel_std', 'rh_Lat_Fis-ant-Vertical_foldind',
    'Left-choroid-plexus_volume', 'lh_G_postcentral_volume', 'rh_S_intrapariet_and_P_trans_foldind',
    'lh_S_postcentral_volume', 'rh_G_parietal_sup_thicknessstd'],
    'categorical': ['gender', 'smoke'],
    'metadata': ['ID', 'center', 'label']
}

VISUAL_CONFIG = {
    'top_n': 5,          # 显示差异最大的前5个特征
    'dpi': 300,          # 图片分辨率
    'figsize': (12, 6),  # 图片尺寸（宽，高）
    'bins': 50           # 直方图分箱数
}

# --------------------- 核心函数 ---------------------
def load_and_validate(path):
    """加载并验证数据集完整性"""
    df = pd.read_csv(path)
    
    # 验证必须存在的列
    required = FEATURE_CONFIG['continuous'] + FEATURE_CONFIG['categorical'] + FEATURE_CONFIG['metadata']
    missing = set(required) - set(df.columns)
    if missing:
        raise KeyError(f"文件 {os.path.basename(path)} 缺失列: {missing}")
    
    return df

def plot_distributions(train_df, external_df):
    """绘制特征分布对比图"""
    # 创建输出目录
    os.makedirs(PATHS['visualization'], exist_ok=True)
    
    # 计算特征差异
    diff = (external_df[FEATURE_CONFIG['continuous']].mean() - train_df[FEATURE_CONFIG['continuous']].mean()).abs()
    top_features = diff.nlargest(VISUAL_CONFIG['top_n']).index.tolist()
    
    print("\n生成特征分布对比图：")
    for feature in top_features:
        plt.figure(figsize=VISUAL_CONFIG['figsize'])
        
        # 绘制双变量分布
        sns.histplot(
            data=pd.concat([
                train_df.assign(Source='train'),
                external_df.assign(Source='external')
            ]),
            x=feature,
            hue='Source',
            element='step',
            stat='density',
            common_norm=False,
            bins=VISUAL_CONFIG['bins'],
            palette=['#1f77b4', '#ff7f0e']
        )
        
        plt.title(f"特征分布对比 - {feature}\n(均值差异: {diff[feature]:.2f} SD)")
        plt.xlabel('标准化值')
        
        # 保存图片
        plot_path = os.path.join(PATHS['visualization'], f'{feature}_distribution.png')
        plt.savefig(plot_path, dpi=VISUAL_CONFIG['dpi'], bbox_inches='tight')
        plt.close()
        print(f"已保存：{plot_path}")

def standardize_dataset():
    """标准化主流程"""
    # 加载数据
    train_df = load_and_validate(PATHS['train'])
    internal_df = load_and_validate(PATHS['internal'])
    external_df = load_and_validate(PATHS['external'])
    
    # 初始化标准化器
    scaler = StandardScaler()
    scaler.fit(train_df[FEATURE_CONFIG['continuous']])
    
    # 保存标准化器
    joblib.dump(scaler, PATHS['scaler'])
    
    # 定义标准化函数
    def apply_scaling(df):
        scaled = df.copy()
        scaled[FEATURE_CONFIG['continuous']] = scaler.transform(df[FEATURE_CONFIG['continuous']])
        return scaled
    
    # 应用标准化
    scaled_train = apply_scaling(train_df)
    scaled_internal = apply_scaling(internal_df)
    scaled_external = apply_scaling(external_df)
    
    # 保存结果
    scaled_train.to_csv(PATHS['output']['train'], index=False)
    scaled_internal.to_csv(PATHS['output']['internal'], index=False)
    scaled_external.to_csv(PATHS['output']['external'], index=False)
    
    return scaled_train, scaled_internal, scaled_external

# --------------------- 主程序 ---------------------
if __name__ == "__main__":
    # 创建输出目录
    for path in PATHS['output'].values():
        os.makedirs(os.path.dirname(path), exist_ok=True)
    
    try:
        # 执行标准化
        train_scaled, _, external_scaled = standardize_dataset()
        
        # 生成可视化
        plot_distributions(train_scaled, external_scaled)
        
        # 打印验证信息
        print("\n标准化验证摘要：")
        print(f"训练集样本量: {len(train_scaled)}")
        print(f"外部验证集样本量: {len(external_scaled)}")
        print(f"\n标准化模型已保存至: {PATHS['scaler']}")
        print(f"可视化结果目录: {PATHS['visualization']}")
        
    except Exception as e:
        print(f"\n错误发生：{str(e)}")
        print("建议检查：")
        print("1. 文件路径是否正确")
        print("2. 特征列名称是否与配置匹配")
        print("3. 是否已安装所有依赖库（matplotlib/seaborn）")
import pandas as pd
import numpy as np
import joblib
import matplotlib.pyplot as plt
from sklearn.model_selection import (
    StratifiedKFold, GridSearchCV, learning_curve
)
from sklearn.metrics import roc_curve, auc
from sklearn.base import clone
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import (
    RandomForestClassifier, ExtraTreesClassifier,
    GradientBoostingClassifier, AdaBoostClassifier
)
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import LogisticRegression
from sklearn.neural_network import MLPClassifier
from xgboost import XGBClassifier
import os

# --------------------- 全局配置 ---------------------
CONFIG = {
    # 数据路径
    "data_path": r"H:\taskPONV\stable\scaled_train.csv",
    
    # 输出配置
    "model_dir": r"H:\taskPONV\stable\models",
    "result_file": r"H:\taskPONV\stable\model_performance.csv",
    "roc_dir": r"H:\taskPONV\stable\roc_curves",
    "learning_curve_dir": r"H:\taskPONV\stable\learning_curves",  # 新增学习曲线目录
    
    # 计算资源
    "n_jobs": 4,          # 并行任务数
    "random_state": 42,   # 随机种子
    
    # 可视化参数
    "roc_figsize": (12, 8),
    "roc_dpi": 300,
    "roc_linewidth": 1,
    "roc_alpha": 0.3,
    "roc_palette": {
        "mean": "#1f77b4",
        "std": "#1f77b480",
        "chance": "#ff7f0e"
    },
    # 新增学习曲线参数
    "lc_figsize": (10, 6),
    "lc_dpi": 300,
    "lc_colors": {
        "train": "#d62728",
        "valid": "#2ca02c"
    }
}

# --------------------- 模型定义 ---------------------
base_models = {
    'SVM': SVC(probability=True, random_state=CONFIG['random_state']),
    'KNN': KNeighborsClassifier(algorithm='kd_tree'),
    'DecisionTree': DecisionTreeClassifier(random_state=CONFIG['random_state']),
    'RandomForest': RandomForestClassifier(random_state=CONFIG['random_state']),
    'ExtraTrees': ExtraTreesClassifier(random_state=CONFIG['random_state']),
    'GradientBoosting': GradientBoostingClassifier(random_state=CONFIG['random_state']),
    'AdaBoost': AdaBoostClassifier(random_state=CONFIG['random_state']),
    'GaussianNB': GaussianNB(),
    'LogisticRegression': LogisticRegression(random_state=CONFIG['random_state']),
    'MLP': MLPClassifier(max_iter=300, random_state=CONFIG['random_state']),
    'XGBoost': XGBClassifier(
        use_label_encoder=False,
        eval_metric='logloss',
        random_state=CONFIG['random_state']
    )
}

param_grids = {
    'SVM': {
        'C': [0.1, 1, 10],
        'gamma': ['scale'],
        'kernel': ['linear', 'rbf'],
    },
    'KNN': {
        'n_neighbors': [3, 5, 7],
        'weights': ['uniform', 'distance'],
        'algorithm': ['auto']
    },
    'DecisionTree': {
        'max_depth': [10, 20, 30],
        'min_samples_split': [2, 10],
        'min_samples_leaf': [1, 4],
        'criterion': ['gini']
    },
    'RandomForest': {
        'n_estimators': [100, 200, 300],
        'max_depth': [10, 30],
        'min_samples_split': [2, 10],
        'min_samples_leaf': [1, 4],
        'bootstrap': [True]
    },
    'ExtraTrees': {
        'n_estimators': [100, 200],
        'max_depth': [10, 30],
        'min_samples_split': [2, 10],
        'min_samples_leaf': [1, 4],
        'bootstrap': [True]
    },
    'GradientBoosting': {
        'n_estimators': [100, 200],
        'learning_rate': [0.01, 0.1],
        'max_depth': [3, 5],
        'min_samples_split': [2, 10],
        'min_samples_leaf': [1, 4],
        'subsample': [0.8, 1.0]
    },
    'AdaBoost': {
        'n_estimators': [100, 200],
        'learning_rate': [0.1, 0.5]
    },
    'GaussianNB': {
        'var_smoothing': [1e-9, 1e-8]
    },
    'LogisticRegression': {
        'C': [0.1, 1, 10],
        'solver': ['lbfgs'],
        'penalty': ['l2'],
        'max_iter': [200]
    },
    'MLP': {
        'hidden_layer_sizes': [(100,), (150, 100)],
        'activation': ['relu'],
        'solver': ['adam'],
        'alpha': [0.0001, 0.001],
        'learning_rate': ['constant'],
        'max_iter': [300]
    },
    'XGBoost': {
        'n_estimators': [100, 200],
        'max_depth': [5, 7],
        'learning_rate': [0.01, 0.1],
        'subsample': [0.8],
        'colsample_bytree': [0.8]
    }
}

# --------------------- 核心函数 ---------------------
def load_data():
    """加载标准化后的训练数据"""
    df = pd.read_csv(CONFIG['data_path'])
    X = df.drop(columns=['label', 'ID', 'center'])
    y = df['label']
    print(f"\n数据加载完成，特征维度: {X.shape}")
    return X, y

def train_and_optimize():
    """模型训练与超参数优化"""
    # 创建输出目录
    os.makedirs(CONFIG['model_dir'], exist_ok=True)
    
    # 加载数据
    X, y = load_data()
    
    # 初始化记录器
    performance = []
    best_estimators = {}

    # 遍历所有模型
    for name in base_models.keys():
        print(f"\n{'='*60}")
        print(f"🔥 正在处理模型: {name}")
        
        try:
            model = base_models[name]
            grid = param_grids.get(name, {})
            
            # 配置并行
            n_jobs = CONFIG['n_jobs'] if name not in ['MLP', 'XGBoost'] else 1
            
            # 网格搜索
            searcher = GridSearchCV(
                estimator=model,
                param_grid=grid,
                cv=StratifiedKFold(n_splits=10, shuffle=True, random_state=CONFIG['random_state']),
                scoring='roc_auc',
                n_jobs=n_jobs,
                verbose=2
            )
            searcher.fit(X, y)
            
            # 记录最佳模型
            best_estimator = searcher.best_estimator_
            best_estimators[name] = best_estimator
            
            # 保存模型
            model_path = os.path.join(CONFIG['model_dir'], f"{name}_best.pkl")
            joblib.dump(best_estimator, model_path)
            
            # 记录性能
            performance.append({
                'Model': name,
                'Best_AUC': searcher.best_score_,
                'Parameters': str(searcher.best_params_),
                'Path': model_path
            })
            print(f"✅ {name} 优化完成 | 最佳AUC: {searcher.best_score_:.4f}")

        except Exception as e:
            print(f"❌ {name} 训练失败: {str(e)}")
            performance.append({
                'Model': name,
                'Best_AUC': np.nan,
                'Parameters': f"Error: {str(e)}",
                'Path': ''
            })
    
    # 保存性能报告
    pd.DataFrame(performance).to_csv(CONFIG['result_file'], index=False)
    return best_estimators, X, y

def visualize_rocs(estimators, X, y):
    """交叉验证ROC曲线可视化"""
    # 创建目录
    os.makedirs(CONFIG['roc_dir'], exist_ok=True)
    
    # 初始化交叉验证器
    cv = StratifiedKFold(n_splits=10, shuffle=True, random_state=CONFIG['random_state'])
    
    for name, estimator in estimators.items():
        print(f"\n📊 正在生成 {name} 的ROC曲线...")
        plt.figure(figsize=CONFIG['roc_figsize'])
        
        # 存储中间结果
        tprs, aucs = [], []
        mean_fpr = np.linspace(0, 1, 100)
        
        # 交叉验证
        for fold, (train_idx, test_idx) in enumerate(cv.split(X, y), 1):
            X_train, X_test = X.iloc[train_idx], X.iloc[test_idx]
            y_train, y_test = y.iloc[train_idx], y.iloc[test_idx]
            
            # 克隆模型保证独立性
            cloned_model = clone(estimator)
            cloned_model.fit(X_train, y_train)
            
            # 预测概率
            if hasattr(cloned_model, "predict_proba"):
                proba = cloned_model.predict_proba(X_test)[:, 1]
            else:
                proba = cloned_model.decision_function(X_test)
            
            # 计算ROC
            fpr, tpr, _ = roc_curve(y_test, proba)
            roc_auc = auc(fpr, tpr)
            
            # 记录结果
            interp_tpr = np.interp(mean_fpr, fpr, tpr)
            interp_tpr[0] = 0.0
            tprs.append(interp_tpr)
            aucs.append(roc_auc)
            
            # 绘制单折曲线
            plt.plot(fpr, tpr, lw=CONFIG['roc_linewidth'],
                     alpha=CONFIG['roc_alpha'],
                     label=f'Fold {fold} (AUC = {roc_auc:.2f})')
        
        # 计算统计量
        mean_tpr = np.mean(tprs, axis=0)
        mean_tpr[-1] = 1.0
        mean_auc = auc(mean_fpr, mean_tpr)
        std_auc = np.std(aucs)
        
        # 绘制平均曲线
        plt.plot(mean_fpr, mean_tpr, color=CONFIG['roc_palette']['mean'],
                 label=f'Mean (AUC = {mean_auc:.2f} ± {std_auc:.2f})',
                 lw=2, alpha=0.8)
        
        # 绘制标准差区域
        std_tpr = np.std(tprs, axis=0)
        plt.fill_between(mean_fpr, 
                         np.maximum(mean_tpr - std_tpr, 0),
                         np.minimum(mean_tpr + std_tpr, 1),
                         color=CONFIG['roc_palette']['std'], alpha=0.2)
        
        # 绘制参考线
        plt.plot([0, 1], [0, 1], linestyle='--', 
                 color=CONFIG['roc_palette']['chance'], 
                 lw=2, alpha=0.8)
        
        # 美化图形
        plt.xlim([-0.05, 1.05])
        plt.ylim([-0.05, 1.05])
        plt.xlabel('False Positive Rate', fontsize=12)
        plt.ylabel('True Positive Rate', fontsize=12)
        plt.title(f'{name} - ROC Curves', fontsize=14)
        plt.legend(loc='lower right', fontsize=9)
        
        # 保存图片
        save_path = os.path.join(CONFIG['roc_dir'], f"{name}_roc.png")
        plt.savefig(save_path, dpi=CONFIG['roc_dpi'], bbox_inches='tight')
        plt.close()
        print(f"💾 已保存: {save_path}")

def plot_learning_curve(estimator, X, y, cv, scoring='roc_auc'):
    """绘制单个模型的学习曲线"""
    train_sizes, train_scores, valid_scores = learning_curve(
        estimator=estimator,
        X=X,
        y=y,
        cv=cv,
        scoring=scoring,
        n_jobs=CONFIG['n_jobs'],
        train_sizes=np.linspace(0.1, 1.0, 10),
        verbose=0
    )
    
    # 计算统计量
    train_scores_mean = np.mean(train_scores, axis=1)
    train_scores_std = np.std(train_scores, axis=1)
    valid_scores_mean = np.mean(valid_scores, axis=1)
    valid_scores_std = np.std(valid_scores, axis=1)

    # 创建画布
    plt.figure(figsize=CONFIG['lc_figsize'])
    
    # 绘制训练曲线
    plt.plot(train_sizes, train_scores_mean, 'o-', 
             color=CONFIG['lc_colors']['train'], 
             label='Training Score')
    plt.fill_between(train_sizes, 
                     train_scores_mean - train_scores_std,
                     train_scores_mean + train_scores_std,
                     alpha=0.1, color=CONFIG['lc_colors']['train'])
    
    # 绘制验证曲线
    plt.plot(train_sizes, valid_scores_mean, 'o-',
             color=CONFIG['lc_colors']['valid'],
             label='Validation Score')
    plt.fill_between(train_sizes,
                     valid_scores_mean - valid_scores_std,
                     valid_scores_mean + valid_scores_std,
                     alpha=0.1, color=CONFIG['lc_colors']['valid'])
    
    # 图形装饰
    plt.xlabel("Training examples", fontsize=12)
    plt.ylabel(scoring.upper(), fontsize=12)
    plt.title("Learning Curve", fontsize=14)
    plt.grid(True, alpha=0.3)
    plt.legend(loc='best')
    return plt

def visualize_learning_curves(estimators, X, y):
    """生成所有模型的学习曲线"""
    os.makedirs(CONFIG['learning_curve_dir'], exist_ok=True)
    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=CONFIG['random_state'])
    
    print("\n" + "="*60)
    print("📈 开始生成学习曲线...")
    
    for name, estimator in estimators.items():
        try:
            print(f"正在生成 {name} 的学习曲线...")
            
            # 绘制曲线
            plt = plot_learning_curve(estimator, X, y, cv)
            
            # 保存图片
            save_path = os.path.join(CONFIG['learning_curve_dir'], f"{name}_lc.png")
            plt.savefig(save_path, dpi=CONFIG['lc_dpi'], bbox_inches='tight')
            plt.close()
            print(f"💾 已保存: {save_path}")
            
        except Exception as e:
            print(f"❌ {name} 学习曲线生成失败: {str(e)}")

# --------------------- 主程序 ---------------------
if __name__ == "__main__":
    # 检查参数网格
    missing = set(base_models.keys()) - set(param_grids.keys())
    if missing:
        print(f"⚠️ 警告: 以下模型缺少参数网格: {missing}")
    
    # 执行训练流程
    best_models, X_data, y_data = train_and_optimize()
    
    # 执行可视化
    print("\n" + "="*60)
    print("🎨 开始生成ROC曲线...")
    visualize_rocs(best_models, X_data, y_data)
    
    print("\n" + "="*60)
    print("📊 开始生成学习曲线...")
    visualize_learning_curves(best_models, X_data, y_data)
    
    print("\n" + "="*60)
    print("✅ 全部流程完成！")
    print(f"模型保存至: {CONFIG['model_dir']}")
    print(f"ROC曲线: {CONFIG['roc_dir']}")
    print(f"学习曲线: {CONFIG['learning_curve_dir']}")
    print(f"性能报告: {CONFIG['result_file']}")
import joblib
import os
import shap
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
from tqdm import tqdm
from sklearn.base import is_classifier

# --------------------- 配置参数 ---------------------
CONFIG = {
    "model_dir": r"H:\taskPONV\stable\models",          # 模型保存路径
    "data_path": r"H:\taskPONV\stable\scaled_train.csv", # 分析用数据集路径
    "shap_dir": r"H:\taskPONV\stable\shap_plots",        # 可视化结果路径
    "sample_size": 100,             # 用于计算的样本量（平衡速度与精度）
    "max_display": 20,              # 显示的特征数量
    "seed": 42                      # 随机种子
}

# --------------------- 核心函数 ---------------------
def load_models():
    """加载所有保存的模型"""
    models = {}
    for fname in os.listdir(CONFIG["model_dir"]):
        if fname.endswith(".pkl"):
            name = fname.replace("_best.pkl", "")
            path = os.path.join(CONFIG["model_dir"], fname)
            models[name] = joblib.load(path)
            print(f"✅ 已加载模型: {name}")
    return models

def prepare_data():
    """准备分析数据"""
    df = pd.read_csv(CONFIG["data_path"])
    X = df.drop(columns=['label', 'ID', 'center'])  # 移除标签和元数据
    return X.sample(n=min(CONFIG["sample_size"], len(X)), 
                   random_state=CONFIG["seed"])

def get_explainer(model, X_sample):
    """自动选择适合的SHAP解释器"""
    try:
        if isinstance(model, (XGBClassifier, 
                            RandomForestClassifier,
                            DecisionTreeClassifier,
                            ExtraTreesClassifier,
                            GradientBoostingClassifier)):
            return shap.TreeExplainer(model)
        elif isinstance(model, (LogisticRegression, 
                               LinearSVC)):
            return shap.LinearExplainer(model, X_sample)
        else:
            return shap.KernelExplainer(model.predict_proba, X_sample)
    except:
        return shap.KernelExplainer(model.predict_proba, X_sample)

def explain_model(name, model, X_sample):
    """执行SHAP解释流程"""
    try:
        # 初始化解释器
        explainer = get_explainer(model, X_sample)
        
        # 计算SHAP值
        shap_values = explainer.shap_values(X_sample)
        
        # 统一格式处理
        if isinstance(shap_values, list):
            shap_values = shap_values[1]  # 取正类的SHAP值
        
        # 可视化分析
        plt.figure(figsize=(10, 8))
        
        # 1. 特征重要性图
        shap.summary_plot(shap_values, X_sample, plot_type="bar", 
                         max_display=CONFIG["max_display"], show=False)
        plt.title(f"{name} - Feature Importance", fontsize=14)
        plt.savefig(os.path.join(CONFIG["shap_dir"], f"{name}_importance.png"), 
                   bbox_inches='tight', dpi=300)
        plt.close()
        
        # 2. 详细分布图
        shap.summary_plot(shap_values, X_sample, 
                         max_display=CONFIG["max_display"], show=False)
        plt.title(f"{name} - SHAP Summary", fontsize=14)
        plt.savefig(os.path.join(CONFIG["shap_dir"], f"{name}_summary.png"),
                   bbox_inches='tight', dpi=300)
        plt.close()
        
        # 3. 单个样本解释（取第一个样本）
        plt.figure(figsize=(12, 6))
        shap.force_plot(explainer.expected_value[1], shap_values[0,:], 
                       X_sample.iloc[0,:], matplotlib=True, show=False)
        plt.title(f"{name} - Individual Explanation", fontsize=14)
        plt.savefig(os.path.join(CONFIG["shap_dir"], f"{name}_individual.png"),
                   bbox_inches='tight', dpi=300)
        plt.close()
        
        print(f"✅ 已完成 {name} 的SHAP分析")
        
    except Exception as e:
        print(f"❌ {name} SHAP分析失败: {str(e)}")

# --------------------- 主程序 ---------------------
if __name__ == "__main__":
    # 创建输出目录
    os.makedirs(CONFIG["shap_dir"], exist_ok=True)
    
    # 加载模型和数据
    models = load_models()
    data_sample = prepare_data()
    
    # 遍历模型进行分析
    print("\n" + "="*60)
    print("🎯 开始SHAP特征解释分析...")
    for name, model in tqdm(models.items(), desc="模型分析进度"):
        if is_classifier(model):
            explain_model(name, model, data_sample)
        else:
            print(f"⚠️ 跳过非分类模型: {name}")
    
    print("\n" + "="*60)
    print(f"✅ 所有分析完成！结果保存至: {CONFIG['shap_dir']}")
    print("生成文件包括：")
    print("- [模型名]_importance.png : 特征重要性排序")
    print("- [模型名]_summary.png    : SHAP值分布图")
    print("- [模型名]_individual.png : 单样本解释图")
from sklearn.metrics import roc_curve, auc
import matplotlib.pyplot as plt
import pandas as pd
import joblib
import xgboost as xgb
# 配置参数
VAL_CONFIG = {
    "internal_path": r"H:\taskPONV\stable\scaled_internaldkt.csv",  # 内部验证集路径
    "external_path": r"H:\taskPONV\stable\scaled_externaldkt.csv",  # 外部验证集路径
    "model_dir": r"H:\taskPONV\stable\modelsdkt",                # 模型目录
    "output_dir": r"H:\taskPONV\stable\val_resultsdkt",          # 结果保存路径
    "dpi": 300,                                               # 图片分辨率
    "figsize": (12, 8)                                        # 图片尺寸
}
SELECTED_FEATURES = ['Left-Pallidum_mean_intensity', 'lh_precuneus_curvind', 'Left-vessel_std', 'lh_precentral_foldind', 'rh_superiorparietal_foldind', 
        'lh_cuneus_volume', 'Right-Thalamus_mean_intensity', 'rh_pericalcarine_meancurv', 'hypertension', 'Left-VentralDC_volume',
        'lh_pericalcarine_foldind', 'rh_cuneus_foldind']
# def load_val_data(data_path):
#     """加载验证数据集"""
#     df = pd.read_csv(data_path)
#     X = df.drop(columns=['label', 'ID', 'center'])  # 根据实际列名调整
#     y = df['label']
#     print(f"验证集 {os.path.basename(data_path)} 加载完成，样本量：{len(X)}")
#     return X, y
def load_val_data(data_path):
    """加载验证数据集（显式选择特征）"""
    df = pd.read_csv(data_path)
    
    # 验证特征完整性
    missing = set(SELECTED_FEATURES) - set(df.columns)
    if missing:
        raise KeyError(f"验证集缺失特征: {missing}")
    
    # 选择指定特征
    X = df[SELECTED_FEATURES]
    y = df['label']
    
    # 处理分类变量（适配XGBoost等模型）
    categoricals = ['gender', 'smoke','PONVhistory','postopioduse']
    for col in categoricals:
        if col in X.columns:
            X[col] = X[col].astype('category')
    
    print(f"验证集 {os.path.basename(data_path)} 加载完成，样本量：{len(X)}")
    return X, y
def plot_roc_curves(estimators, X, y, title_suffix, data_df):
    """绘制ROC曲线并保存预测结果"""
    plt.figure(figsize=VAL_CONFIG["figsize"])
    
    # 创建结果存储DataFrame
    results_df = data_df[['ID']].copy()
    results_df['label'] = y.values
    
    for model_name in estimators.keys():
        try:
            model = estimators[model_name]
            
            # 统一使用scikit-learn接口进行预测
            if hasattr(model, 'feature_names_in_'):
                # 确保特征顺序与训练时一致
                X_ordered = X[model.feature_names_in_]
            else:
                X_ordered = X
                
            # 特殊处理分类特征
            if 'XGBoost' in model_name:
                # 转换分类列为category类型
                categoricals = ['gender', 'smoke','PONVhistory','postopioduse']
                for col in categoricals:
                    if col in X_ordered.columns:
                        X_ordered = X_ordered.copy()
                        X_ordered[col] = X_ordered[col].astype('category')
                
                # 使用原生predict方法
                y_proba = model.predict_proba(X_ordered)[:, 1]
                y_pred = model.predict(X_ordered)
            else:
                y_proba = model.predict_proba(X_ordered)[:, 1]
                y_pred = model.predict(X_ordered)
            
            # 存储预测结果
            results_df[f'{model_name}_prob'] = y_proba
            results_df[f'{model_name}_pred'] = y_pred
            
            # 计算ROC指标
            fpr, tpr, _ = roc_curve(y, y_proba)
            roc_auc = auc(fpr, tpr)
            
            # 绘制曲线
            plt.plot(fpr, tpr, linewidth=2, 
                    label=f'{model_name} (AUC = {roc_auc:.2f})')
            
        except Exception as e:
            print(f"❌ {model_name} 验证失败: {str(e)}")
            continue
    
    # 保存预测结果
    output_path = os.path.join(VAL_CONFIG["output_dir"], f"{title_suffix}_predictions.csv")
    results_df.to_csv(output_path, index=False)
    print(f"✅ 已保存预测结果: {output_path}")
    
    # 装饰图形（保持原有绘图代码不变）
    plt.plot([0, 1], [0, 1], color='navy', linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate', fontsize=12)
    plt.ylabel('True Positive Rate', fontsize=12)
    plt.title(f'{title_suffix} ROC Curve', fontsize=14)
    plt.legend(loc='lower right', fontsize=9)
    
    # 保存图片
    save_path = os.path.join(VAL_CONFIG["output_dir"], f"{title_suffix}_roc.png")
    plt.savefig(save_path, dpi=VAL_CONFIG["dpi"], bbox_inches='tight')
    plt.close()

def load_val_data(data_path):
    """加载验证数据集（保留ID列）"""
    df = pd.read_csv(data_path)
    
    # 验证特征完整性
    missing = set(SELECTED_FEATURES) - set(df.columns)
    if missing:
        raise KeyError(f"验证集缺失特征: {missing}")
    
    # 选择指定特征
    X = df[SELECTED_FEATURES]
    y = df['label']
    
    # 处理分类变量
    categoricals = ['gender', 'smoke']
    for col in categoricals:
        if col in X.columns:
            X[col] = X[col].astype('category')
    
    print(f"验证集 {os.path.basename(data_path)} 加载完成，样本量：{len(X)}")
    return X, y, df[['ID'] + SELECTED_FEATURES + ['label']]  # 返回完整数据用于保存结果

def evaluate_models():
    """主评估流程"""
    os.makedirs(VAL_CONFIG["output_dir"], exist_ok=True)
    
    # 加载模型
    estimators = {}
    for fname in os.listdir(VAL_CONFIG["model_dir"]):
        if fname.endswith(".pkl"):
            name = fname.replace("_best.pkl", "")
            path = os.path.join(VAL_CONFIG["model_dir"], fname)
            estimators[name] = joblib.load(path)
    
    # 内部验证评估
    print("\n" + "="*60)
    print("📊 开始内部验证集评估...")
    X_val_int, y_val_int, df_int = load_val_data(VAL_CONFIG["internal_path"])
    plot_roc_curves(estimators, X_val_int, y_val_int, "Test_cohort", df_int)
    
    # 外部验证评估
    print("\n" + "="*60)
    print("🌍 开始外部验证集评估...")
    X_val_ext, y_val_ext, df_ext = load_val_data(VAL_CONFIG["external_path"])
    plot_roc_curves(estimators, X_val_ext, y_val_ext, "External_cohort", df_ext)
if __name__ == "__main__":
    evaluate_models()
    print("\n" + "="*60)
    print("✅ 全部验证完成！结果保存至:", VAL_CONFIG["output_dir"])