{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a31c6e51-7360-447e-ab07-53c51d3b2204",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "9bbe44db-0f41-4543-98e6-1342433545a6",
   "metadata": {},
   "source": [
    "# Response and Answer whether there has been a data leak"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5beaded8-a60b-4e4f-9a31-a95d4fdaa83b",
   "metadata": {},
   "source": [
    "We strictly divided the dataset according to the requirements and the description in the paper. It's saved in FASTA format. Now we check whether there is any overlap between the training set and the test set, and there isn't, so there's no issue of test set data leaking into the training set."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d845939-d7c5-4655-a76d-a37c80b6ff33",
   "metadata": {},
   "source": [
    "## Data Download"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53cba1be-8003-490f-8674-6c764cbc25cf",
   "metadata": {},
   "source": [
    "All the data for this notebook can be downloaded from http://public.aibiochem.net/proteins/CSRPsvm/"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5981542c-da01-497a-863d-d0b660de874e",
   "metadata": {},
   "source": [
    "## Define a function to convert fasta to DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b9415ce4-121f-48b7-a4e8-ade78b4b6ac7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from typing import Tuple\n",
    "\n",
    "def fasta_to_dataframe(fasta_path: str) -> Tuple[pd.DataFrame, int]:\n",
    "    \"\"\"\n",
    "    Read a FASTA file and return a DataFrame with 'id' and 'sequence' columns,\n",
    "    along with the total number of sequence entries.\n",
    "\n",
    "    Parameters:\n",
    "        fasta_path (str): Path to the FASTA file.\n",
    "\n",
    "    Returns:\n",
    "        tuple: (pd.DataFrame, int)\n",
    "            - DataFrame: one row per sequence, with columns 'id' and 'sequence'\n",
    "            - int: number of sequence entries\n",
    "\n",
    "    Raises:\n",
    "        FileNotFoundError: If the file does not exist.\n",
    "        ValueError: If no valid sequence is found in the FASTA file.\n",
    "    \"\"\"\n",
    "    ids = []\n",
    "    sequences = []\n",
    "    current_id = None\n",
    "    current_seq_lines = []\n",
    "\n",
    "    try:\n",
    "        with open(fasta_path, 'r') as f:\n",
    "            for line in f:\n",
    "                line = line.strip()\n",
    "                if not line:\n",
    "                    continue  # skip empty lines\n",
    "\n",
    "                if line.startswith('>'):\n",
    "                    # New header found; save the previous sequence if present\n",
    "                    if current_id is not None:\n",
    "                        seq = ''.join(current_seq_lines)\n",
    "                        if seq:  # ignore empty sequences\n",
    "                            ids.append(current_id)\n",
    "                            sequences.append(seq)\n",
    "\n",
    "                    # Start a new sequence\n",
    "                    current_id = line[1:]  # remove leading '>'\n",
    "                    current_seq_lines = []\n",
    "                else:\n",
    "                    # Sequence line\n",
    "                    current_seq_lines.append(line)\n",
    "\n",
    "            # Process the last sequence\n",
    "            if current_id is not None:\n",
    "                seq = ''.join(current_seq_lines)\n",
    "                if seq:\n",
    "                    ids.append(current_id)\n",
    "                    sequences.append(seq)\n",
    "\n",
    "    except FileNotFoundError:\n",
    "        raise FileNotFoundError(f\"FASTA file not found: {fasta_path}\")\n",
    "\n",
    "    if len(ids) == 0:\n",
    "        raise ValueError(\"No valid sequence entries found in the FASTA file.\")\n",
    "\n",
    "    df = pd.DataFrame({'id': ids, 'sequence': sequences})\n",
    "    entry_count = len(df)\n",
    "\n",
    "    print(f\"Successfully read FASTA file. Total sequences: {entry_count}\")\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25c86aef-eb33-4b23-bfa9-0cef47c1f860",
   "metadata": {},
   "source": [
    "## Read the training set FASTA and convert it to a DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5075f38c-fbc8-42b5-96b5-2a1816034ca1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully read FASTA file. Total sequences: 60843\n",
      "(60843, 2)\n",
      "                                                  id  \\\n",
      "0  pos_inVivo_gene=CCL1_entry=UPI0007C0E151_file=...   \n",
      "1  pos_CellAge_gene=AAK1_entry=UPI00028069D4_file...   \n",
      "2  neg_Cytoskeletal_gene=DES_entry=UPI00119B29EA_...   \n",
      "3  pos_inVivo_gene=PLAT_entry=UPI000D669C99_file=...   \n",
      "4  neg_Cytoskeletal_gene=MAP2_entry=UPI00023462C8...   \n",
      "\n",
      "                                            sequence  \n",
      "0  MTSQTSEPRKSKVVSDDDLYRNSSQYRLWSFTKQQLLDRRAECHNR...  \n",
      "1  MILMEFLVVLGCLLLGTRFGGMGLGLISGIGLFLLTFVFGLVPGEP...  \n",
      "2  MAAAATQRPLDAFSAPHLHSNPTIEVNKPTKSSYTPRDVTTVFNYF...  \n",
      "3  MKKTILKSLVATITGVLFITCLSSCSKKGDDDPSPESGERTAKFTV...  \n",
      "4  MSVTIKSTREIDLMREAGRILAITHEELAKAVRPGISTMDIDRLGE...  \n"
     ]
    }
   ],
   "source": [
    "## Training dataset \n",
    "\n",
    "TrainData_pd=fasta_to_dataframe(\"./Repi30PN.Train.fa\")\n",
    "print(TrainData_pd.shape)\n",
    "print(TrainData_pd.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33853eb6-bdf2-4b9e-bb8f-c3d9b4c93335",
   "metadata": {},
   "source": [
    "## Read the testing set FASTA and convert it to a DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c365588a-904e-4c61-959e-b601620cf1b4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully read FASTA file. Total sequences: 15211\n",
      "(15211, 2)\n",
      "                                                  id  \\\n",
      "0  neg_Ribosomal_gene=RPS8_entry=UPI000D223254_fi...   \n",
      "1  pos_CellAge_gene=SYK_entry=UPI0002276745_file=...   \n",
      "2  pos_inVivo_gene=PLAT_entry=UPI000B886BFD_file=...   \n",
      "3  neg_Housekeeping_gene=TBP_entry=UPI001101CFD8_...   \n",
      "4  pos_CellAge_gene=CKB_entry=UPI0006C5B195_file=...   \n",
      "\n",
      "                                            sequence  \n",
      "0  MINDPIADLFTRIRNASLVKCSKVAVPLTQTNERIVQIFAQHGLIV...  \n",
      "1  MSQNLILIVVALLVAVLVIWLLRSRSGQAGHHVDSSATAVGAASET...  \n",
      "2  MVDFSIRMKNKLRFSTCDAPHVPTSKTHEEIILVELRGDLLMITAL...  \n",
      "3  AHEQLPAQRRGGRAGREDRGLDGAQARGGVSPDGSAALTIGDFRSV...  \n",
      "4  MMVSTSNDEEKLISTTDKYFIEQRNIVLQEINETMNSILNGLNGLN...  \n"
     ]
    }
   ],
   "source": [
    "## Testing Dataset\n",
    "TestData_pd=fasta_to_dataframe(\"./Repi30PN.Test.fa\")\n",
    "print(TestData_pd.shape)\n",
    "print(TestData_pd.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3264cbb0-80ae-4dcd-a53f-5145be47ff0f",
   "metadata": {},
   "source": [
    "##  Take the intersection of protein sequences from the training set and the test set"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "09037f30-272a-41d2-af02-003bb1e6098b",
   "metadata": {},
   "source": [
    "(1) Returning an empty set indicates that there is no issue of test set data leaking into the training set.\n",
    "(2) This test set is the one we use for the subsequent evaluation in the paper, and it has no overlap with the training set or the validation sets from k-fold cross-validation on the training set.\n",
    "(3) All model training is done only on the training set, so the models never see the test set data, meaning there's no data leakage issue."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c755e77a-f6c4-43c7-bf2d-338f3390ddde",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Empty DataFrame\n",
      "Columns: [id_x, sequence, id_y]\n",
      "Index: []\n"
     ]
    }
   ],
   "source": [
    "merged = pd.merge(TrainData_pd, TestData_pd, on='sequence', how='inner')\n",
    "print(merged)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab8b4787-a562-44b9-88c3-b1ff200296b8",
   "metadata": {},
   "source": [
    "# Response and Answer whether feature selection causes data leakage"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c77217df-60a9-4b49-980f-52e80fa837fd",
   "metadata": {},
   "source": [
    "Feature selection uses the LightGBM algorithm to rank the importance of features in the training data, let's call it Rank_train. That means the independent test set data doesn't participate in the LightGBM ranking. For independent evaluation, the features in the test set are ordered based on the already established Rank_train order, without re-ranking. The specific steps are shown in the code below."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e25d62fc-330b-43e8-9b51-01cf52c27cc6",
   "metadata": {},
   "source": [
    "## LightGBM's Feature Selection Algorithm - Only for the Training Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c2d87df2-f2f9-4a83-ac00-836661ca08a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import lightgbm as lgb\n",
    "\n",
    "def feature_importance_rank_and_save(train_csv_path, n_estimators, output_train_path=None):\n",
    "    \"\"\"\n",
    "    Compute feature importance using LightGBM, rank features by importance,\n",
    "    and save the reordered training set (features sorted by importance).\n",
    "\n",
    "    Parameters:\n",
    "        train_csv_path (str): Path to the training CSV file.\n",
    "        n_estimators (int): Number of boosting iterations (trees).\n",
    "        output_train_path (str, optional): Path for the reordered training CSV.\n",
    "            If None, a default name is generated automatically.\n",
    "\n",
    "    Returns:\n",
    "        pd.DataFrame: Feature importance table with columns\n",
    "            ['feature', 'importance', 'rank'], sorted by importance (descending).\n",
    "    \"\"\"\n",
    "\n",
    "    # ===================== 1. Load data =====================\n",
    "    print(\"Loading data...\")\n",
    "    train_df = pd.read_csv(train_csv_path)\n",
    "\n",
    "    # Validate data format\n",
    "    if train_df.shape[1] < 3:\n",
    "        raise ValueError(\n",
    "            \"Invalid data format: at least an ID column, a label column, \"\n",
    "            \"and one feature column are required.\"\n",
    "        )\n",
    "\n",
    "    # Split ID, label, and features\n",
    "    # Convention: 1st column = ID, 2nd column = label, 3rd+ columns = features\n",
    "    id_col = train_df.columns[0]       # ID column name\n",
    "    label_col = train_df.columns[1]    # Label column name\n",
    "    feature_cols = train_df.columns[2:]  # Feature column names\n",
    "\n",
    "    X_train = train_df[feature_cols]\n",
    "    y_train = train_df[label_col]\n",
    "\n",
    "    # ===================== 2. Train LightGBM model =====================\n",
    "    print(f\"Training LightGBM model with n_estimators = {n_estimators} ...\")\n",
    "\n",
    "    # Base parameters (automatically detect classification vs. regression)\n",
    "    params = {\n",
    "        \"boosting_type\": \"gbdt\",\n",
    "        \"objective\": \"binary\" if len(y_train.unique()) <= 2 else \"regression\",\n",
    "        \"metric\": \"auc\",\n",
    "        \"random_state\": 42,\n",
    "        \"verbose\": -1,\n",
    "        \"n_estimators\": n_estimators,\n",
    "    }\n",
    "\n",
    "    # Initialize and train the model\n",
    "    if params[\"objective\"] == \"binary\":\n",
    "        lgb_model = lgb.LGBMClassifier(**params)\n",
    "    else:\n",
    "        lgb_model = lgb.LGBMRegressor(**params)\n",
    "\n",
    "    lgb_model.fit(X_train, y_train)\n",
    "\n",
    "    # ===================== 3. Compute and rank feature importance =====================\n",
    "    print(\"Ranking feature importance...\")\n",
    "    feature_importance = pd.DataFrame({\n",
    "        \"feature\": feature_cols,\n",
    "        \"importance\": lgb_model.feature_importances_,\n",
    "    })\n",
    "\n",
    "    # Sort by importance in descending order (most → least important)\n",
    "    feature_importance_sorted = feature_importance.sort_values(\n",
    "        by=\"importance\", ascending=False\n",
    "    ).reset_index(drop=True)\n",
    "    feature_importance_sorted[\"rank\"] = range(1, len(feature_importance_sorted) + 1)\n",
    "\n",
    "    # Print ranking\n",
    "    print(\"\\n===== Feature Importance Ranking (most → least important) =====\")\n",
    "    print(feature_importance_sorted.to_string(index=False))\n",
    "\n",
    "    # Ordered list of features by importance\n",
    "    sorted_feature_cols = feature_importance_sorted[\"feature\"].tolist()\n",
    "\n",
    "    # ===================== 4. Auto-generate output file names =====================\n",
    "    if output_train_path is None:\n",
    "        output_train_path = f\"Train_SFbyLGBM_{n_estimators}.csv\"\n",
    "\n",
    "    # ===================== 5. Save the reordered dataset =====================\n",
    "    print(\"\\nSaving the reordered dataset...\")\n",
    "    new_columns = [id_col, label_col] + sorted_feature_cols\n",
    "    ranked_train_df = train_df[new_columns]\n",
    "    ranked_train_df.to_csv(output_train_path, index=False, encoding=\"utf-8-sig\")\n",
    "    print(f\"Train saved: {output_train_path}\")\n",
    "\n",
    "    # ===================== 6. Save the feature importance table separately =====================\n",
    "    feature_output_path = f\"FeatureImportance_{n_estimators}.csv\"\n",
    "    feature_importance_sorted.to_csv(feature_output_path, index=False, encoding=\"utf-8-sig\")\n",
    "    print(f\"Feature importance table saved: {feature_output_path}\")\n",
    "\n",
    "    return feature_importance_sorted"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a16c6ed2-f68f-47d7-9006-4365bfd0c9a2",
   "metadata": {},
   "source": [
    "## Load the training set and rank the features"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7cc372fe-4c57-4f5d-b8d7-946fc431d189",
   "metadata": {},
   "source": [
    "Since using ESM2 to convert and extract features from proteins is quite time-consuming, we're using the feature CSV files we've already extracted before."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a420ea48-2437-4176-8e02-4625e9448b67",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60843, 1282)\n",
      "                                                  ID  Class    esm_F1  \\\n",
      "0  pos_inVivo_gene=CCL1_entry=UPI0007C0E151_file=...      1 -0.020918   \n",
      "1  pos_CellAge_gene=AAK1_entry=UPI00028069D4_file...      1  0.000364   \n",
      "2  neg_Cytoskeletal_gene=DES_entry=UPI00119B29EA_...      0 -0.111511   \n",
      "3  pos_inVivo_gene=PLAT_entry=UPI000D669C99_file=...      1 -0.008683   \n",
      "4  neg_Cytoskeletal_gene=MAP2_entry=UPI00023462C8...      0  0.031492   \n",
      "\n",
      "     esm_F2    esm_F3    esm_F4    esm_F5    esm_F6    esm_F7    esm_F8  ...  \\\n",
      "0 -0.043912 -0.001424  0.009036 -0.011712 -0.012013  0.015271 -0.086599  ...   \n",
      "1 -0.018648 -0.022993  0.045800 -0.051272  0.012483  0.139637 -0.163522  ...   \n",
      "2 -0.084338  0.008745 -0.021599 -0.007999 -0.016545  0.008820 -0.104619  ...   \n",
      "3  0.065736  0.000272  0.109800 -0.037233  0.039990  0.035683 -0.170515  ...   \n",
      "4 -0.043774 -0.014221  0.021351 -0.075682 -0.048555  0.037727 -0.079231  ...   \n",
      "\n",
      "   esm_F1271  esm_F1272  esm_F1273  esm_F1274  esm_F1275  esm_F1276  \\\n",
      "0   0.056945   0.008526   0.014277   0.011697  -0.047034   0.004576   \n",
      "1   0.008939  -0.109501   0.010417  -0.071428  -0.016907   0.070792   \n",
      "2   0.041293  -0.097168  -0.012764  -0.051015  -0.082216   0.027791   \n",
      "3   0.063256   0.007484  -0.023712  -0.022364  -0.049004  -0.068860   \n",
      "4  -0.012602  -0.162691  -0.061705   0.013504  -0.027724   0.022397   \n",
      "\n",
      "   esm_F1277  esm_F1278  esm_F1279  esm_F1280  \n",
      "0   0.004843  -0.091689  -0.016501   0.037668  \n",
      "1   0.069470  -0.159428  -0.031141   0.054698  \n",
      "2   0.049143  -0.247077   0.009582  -0.004942  \n",
      "3   0.033442  -0.049920  -0.034956   0.004648  \n",
      "4   0.091501  -0.178170   0.005235   0.054448  \n",
      "\n",
      "[5 rows x 1282 columns]\n"
     ]
    }
   ],
   "source": [
    "TrainData_ESM=pd.read_csv(\"./esm2_t33_650M_UR50D.Train.m7.csv\",header=0)\n",
    "print(TrainData_ESM.shape)\n",
    "print(TrainData_ESM.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "79e67268-942b-43e8-8688-a1af7f7a6259",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data...\n",
      "Training LightGBM model with n_estimators = 2560 ...\n",
      "Ranking feature importance...\n",
      "\n",
      "===== Feature Importance Ranking (most → least important) =====\n",
      "  feature  importance  rank\n",
      " esm_F296         162     1\n",
      " esm_F969         151     2\n",
      " esm_F990         148     3\n",
      "esm_F1140         135     4\n",
      "esm_F1146         135     5\n",
      " esm_F496         130     6\n",
      " esm_F538         125     7\n",
      " esm_F452         125     8\n",
      " esm_F710         122     9\n",
      " esm_F292         118    10\n",
      " esm_F342         117    11\n",
      " esm_F144         116    12\n",
      "esm_F1246         112    13\n",
      " esm_F517         111    14\n",
      " esm_F295         110    15\n",
      " esm_F615         110    16\n",
      " esm_F878         110    17\n",
      " esm_F866         110    18\n",
      "esm_F1186         109    19\n",
      " esm_F696         105    20\n",
      " esm_F247         104    21\n",
      " esm_F386         103    22\n",
      "esm_F1154         103    23\n",
      "  esm_F90         101    24\n",
      " esm_F588         101    25\n",
      "esm_F1124         101    26\n",
      " esm_F607         100    27\n",
      " esm_F449         100    28\n",
      " esm_F784         100    29\n",
      " esm_F283          99    30\n",
      "  esm_F71          97    31\n",
      "esm_F1275          97    32\n",
      "esm_F1098          97    33\n",
      " esm_F669          96    34\n",
      " esm_F639          95    35\n",
      " esm_F978          95    36\n",
      " esm_F397          95    37\n",
      " esm_F772          95    38\n",
      " esm_F375          95    39\n",
      "esm_F1017          94    40\n",
      " esm_F390          94    41\n",
      " esm_F322          94    42\n",
      "esm_F1023          94    43\n",
      "esm_F1047          93    44\n",
      " esm_F876          93    45\n",
      " esm_F672          93    46\n",
      "  esm_F35          93    47\n",
      " esm_F608          93    48\n",
      "esm_F1211          92    49\n",
      " esm_F685          92    50\n",
      " esm_F191          92    51\n",
      "esm_F1053          92    52\n",
      " esm_F951          92    53\n",
      " esm_F212          92    54\n",
      " esm_F848          91    55\n",
      "esm_F1200          91    56\n",
      " esm_F776          90    57\n",
      " esm_F166          90    58\n",
      "  esm_F76          90    59\n",
      " esm_F823          90    60\n",
      "esm_F1176          90    61\n",
      "   esm_F3          90    62\n",
      " esm_F786          89    63\n",
      " esm_F201          89    64\n",
      " esm_F198          89    65\n",
      " esm_F141          89    66\n",
      " esm_F183          89    67\n",
      "  esm_F17          89    68\n",
      "esm_F1236          89    69\n",
      " esm_F407          89    70\n",
      " esm_F992          89    71\n",
      "esm_F1245          88    72\n",
      "esm_F1036          88    73\n",
      " esm_F478          88    74\n",
      " esm_F834          88    75\n",
      "esm_F1222          88    76\n",
      "esm_F1052          88    77\n",
      " esm_F587          87    78\n",
      "esm_F1126          87    79\n",
      "esm_F1273          87    80\n",
      "  esm_F11          87    81\n",
      " esm_F634          87    82\n",
      " esm_F585          87    83\n",
      " esm_F681          86    84\n",
      "esm_F1139          86    85\n",
      "esm_F1024          86    86\n",
      " esm_F708          85    87\n",
      " esm_F671          85    88\n",
      " esm_F477          85    89\n",
      " esm_F963          85    90\n",
      " esm_F209          85    91\n",
      "esm_F1136          85    92\n",
      " esm_F831          84    93\n",
      " esm_F425          84    94\n",
      " esm_F557          84    95\n",
      " esm_F513          84    96\n",
      " esm_F574          84    97\n",
      " esm_F707          84    98\n",
      "  esm_F65          84    99\n",
      " esm_F231          84   100\n",
      "  esm_F44          84   101\n",
      " esm_F568          83   102\n",
      " esm_F439          83   103\n",
      " esm_F858          83   104\n",
      " esm_F857          83   105\n",
      " esm_F165          83   106\n",
      " esm_F783          83   107\n",
      "esm_F1115          83   108\n",
      " esm_F879          83   109\n",
      "esm_F1272          82   110\n",
      "esm_F1151          82   111\n",
      " esm_F966          82   112\n",
      " esm_F226          82   113\n",
      "esm_F1081          82   114\n",
      " esm_F301          82   115\n",
      "esm_F1238          82   116\n",
      " esm_F106          82   117\n",
      "esm_F1086          82   118\n",
      " esm_F261          82   119\n",
      " esm_F194          81   120\n",
      " esm_F381          81   121\n",
      " esm_F459          81   122\n",
      " esm_F275          81   123\n",
      " esm_F351          81   124\n",
      " esm_F958          81   125\n",
      "esm_F1000          81   126\n",
      " esm_F706          81   127\n",
      " esm_F124          81   128\n",
      "esm_F1217          81   129\n",
      "esm_F1107          81   130\n",
      "esm_F1113          81   131\n",
      "esm_F1131          80   132\n",
      " esm_F794          80   133\n",
      " esm_F111          80   134\n",
      " esm_F522          80   135\n",
      " esm_F171          80   136\n",
      " esm_F395          80   137\n",
      "  esm_F14          80   138\n",
      "  esm_F81          80   139\n",
      " esm_F885          80   140\n",
      " esm_F514          80   141\n",
      " esm_F870          79   142\n",
      "esm_F1046          79   143\n",
      " esm_F366          79   144\n",
      " esm_F519          79   145\n",
      "esm_F1123          79   146\n",
      " esm_F873          79   147\n",
      " esm_F915          79   148\n",
      " esm_F651          79   149\n",
      " esm_F725          79   150\n",
      "esm_F1229          79   151\n",
      " esm_F135          78   152\n",
      " esm_F169          78   153\n",
      " esm_F835          78   154\n",
      " esm_F468          78   155\n",
      " esm_F998          78   156\n",
      " esm_F775          78   157\n",
      "  esm_F77          78   158\n",
      " esm_F249          78   159\n",
      " esm_F911          78   160\n",
      "esm_F1090          78   161\n",
      " esm_F575          78   162\n",
      "  esm_F63          78   163\n",
      " esm_F960          78   164\n",
      "esm_F1188          77   165\n",
      "esm_F1178          77   166\n",
      " esm_F731          77   167\n",
      "  esm_F27          77   168\n",
      " esm_F127          77   169\n",
      " esm_F333          77   170\n",
      " esm_F914          77   171\n",
      " esm_F839          77   172\n",
      " esm_F767          77   173\n",
      "esm_F1223          77   174\n",
      " esm_F371          77   175\n",
      "   esm_F6          77   176\n",
      " esm_F274          77   177\n",
      " esm_F970          76   178\n",
      " esm_F854          76   179\n",
      "esm_F1013          76   180\n",
      "esm_F1028          76   181\n",
      " esm_F432          76   182\n",
      " esm_F904          76   183\n",
      "esm_F1192          76   184\n",
      "esm_F1185          76   185\n",
      "esm_F1075          76   186\n",
      " esm_F306          76   187\n",
      " esm_F411          76   188\n",
      "esm_F1069          76   189\n",
      "  esm_F93          76   190\n",
      " esm_F664          76   191\n",
      " esm_F683          76   192\n",
      " esm_F659          76   193\n",
      " esm_F724          76   194\n",
      " esm_F368          76   195\n",
      " esm_F571          76   196\n",
      " esm_F771          76   197\n",
      " esm_F954          76   198\n",
      "esm_F1269          76   199\n",
      "esm_F1012          75   200\n",
      "   esm_F5          75   201\n",
      " esm_F357          75   202\n",
      " esm_F289          75   203\n",
      " esm_F458          75   204\n",
      " esm_F416          75   205\n",
      " esm_F388          75   206\n",
      " esm_F733          75   207\n",
      " esm_F965          75   208\n",
      " esm_F331          75   209\n",
      " esm_F203          75   210\n",
      " esm_F570          75   211\n",
      "esm_F1087          75   212\n",
      " esm_F795          75   213\n",
      " esm_F892          74   214\n",
      " esm_F427          74   215\n",
      " esm_F253          74   216\n",
      " esm_F281          74   217\n",
      "esm_F1163          74   218\n",
      " esm_F497          74   219\n",
      " esm_F545          74   220\n",
      " esm_F896          74   221\n",
      " esm_F404          74   222\n",
      " esm_F205          73   223\n",
      " esm_F933          73   224\n",
      " esm_F158          73   225\n",
      " esm_F504          73   226\n",
      " esm_F543          73   227\n",
      "  esm_F74          73   228\n",
      " esm_F120          73   229\n",
      " esm_F985          73   230\n",
      " esm_F591          73   231\n",
      " esm_F204          73   232\n",
      " esm_F673          73   233\n",
      " esm_F662          73   234\n",
      " esm_F405          73   235\n",
      " esm_F830          73   236\n",
      " esm_F222          73   237\n",
      " esm_F131          73   238\n",
      " esm_F470          73   239\n",
      " esm_F110          72   240\n",
      " esm_F643          72   241\n",
      "esm_F1187          72   242\n",
      " esm_F254          72   243\n",
      " esm_F208          72   244\n",
      "esm_F1142          72   245\n",
      "  esm_F19          72   246\n",
      " esm_F875          72   247\n",
      " esm_F303          72   248\n",
      " esm_F586          72   249\n",
      " esm_F345          72   250\n",
      " esm_F140          72   251\n",
      " esm_F759          72   252\n",
      " esm_F185          72   253\n",
      " esm_F238          72   254\n",
      " esm_F889          72   255\n",
      " esm_F765          72   256\n",
      "esm_F1204          72   257\n",
      " esm_F273          71   258\n",
      " esm_F328          71   259\n",
      "  esm_F12          71   260\n",
      "  esm_F51          71   261\n",
      " esm_F648          71   262\n",
      " esm_F618          71   263\n",
      " esm_F339          71   264\n",
      "esm_F1244          71   265\n",
      "  esm_F32          71   266\n",
      " esm_F323          71   267\n",
      " esm_F595          71   268\n",
      "  esm_F58          71   269\n",
      " esm_F252          71   270\n",
      " esm_F139          71   271\n",
      " esm_F525          71   272\n",
      " esm_F716          71   273\n",
      " esm_F503          71   274\n",
      " esm_F714          71   275\n",
      "  esm_F95          71   276\n",
      "esm_F1063          71   277\n",
      " esm_F101          71   278\n",
      " esm_F441          71   279\n",
      " esm_F647          70   280\n",
      " esm_F293          70   281\n",
      "  esm_F26          70   282\n",
      "esm_F1260          70   283\n",
      " esm_F704          70   284\n",
      " esm_F905          70   285\n",
      "esm_F1172          70   286\n",
      "esm_F1256          70   287\n",
      " esm_F487          70   288\n",
      " esm_F480          70   289\n",
      " esm_F787          70   290\n",
      "esm_F1120          70   291\n",
      "  esm_F38          70   292\n",
      " esm_F682          70   293\n",
      " esm_F383          70   294\n",
      "esm_F1040          70   295\n",
      "esm_F1003          70   296\n",
      " esm_F921          70   297\n",
      " esm_F623          70   298\n",
      " esm_F865          69   299\n",
      " esm_F533          69   300\n",
      " esm_F159          69   301\n",
      " esm_F944          69   302\n",
      " esm_F697          69   303\n",
      " esm_F755          69   304\n",
      " esm_F720          69   305\n",
      " esm_F145          69   306\n",
      " esm_F949          69   307\n",
      " esm_F661          69   308\n",
      " esm_F337          69   309\n",
      " esm_F521          69   310\n",
      " esm_F746          69   311\n",
      " esm_F143          69   312\n",
      " esm_F802          69   313\n",
      " esm_F347          69   314\n",
      " esm_F448          69   315\n",
      " esm_F936          69   316\n",
      " esm_F315          69   317\n",
      " esm_F308          69   318\n",
      "  esm_F87          69   319\n",
      "  esm_F34          69   320\n",
      "esm_F1011          69   321\n",
      "esm_F1194          69   322\n",
      " esm_F409          69   323\n",
      "esm_F1196          69   324\n",
      "  esm_F80          69   325\n",
      "esm_F1251          69   326\n",
      " esm_F493          69   327\n",
      " esm_F494          69   328\n",
      " esm_F402          69   329\n",
      " esm_F843          69   330\n",
      "esm_F1001          68   331\n",
      " esm_F377          68   332\n",
      "esm_F1038          68   333\n",
      " esm_F430          68   334\n",
      "  esm_F85          68   335\n",
      " esm_F687          68   336\n",
      " esm_F352          68   337\n",
      " esm_F136          68   338\n",
      " esm_F918          68   339\n",
      "  esm_F37          68   340\n",
      "esm_F1129          68   341\n",
      "  esm_F86          68   342\n",
      " esm_F743          68   343\n",
      "esm_F1114          68   344\n",
      "esm_F1088          68   345\n",
      " esm_F335          68   346\n",
      "  esm_F94          68   347\n",
      " esm_F311          68   348\n",
      " esm_F973          68   349\n",
      "esm_F1208          68   350\n",
      " esm_F744          67   351\n",
      " esm_F391          67   352\n",
      " esm_F547          67   353\n",
      " esm_F396          67   354\n",
      " esm_F403          67   355\n",
      " esm_F845          67   356\n",
      " esm_F505          67   357\n",
      " esm_F579          67   358\n",
      " esm_F788          67   359\n",
      " esm_F631          67   360\n",
      " esm_F625          67   361\n",
      " esm_F511          67   362\n",
      " esm_F515          67   363\n",
      " esm_F642          67   364\n",
      " esm_F614          67   365\n",
      " esm_F641          67   366\n",
      " esm_F189          67   367\n",
      " esm_F180          67   368\n",
      " esm_F119          67   369\n",
      " esm_F130          67   370\n",
      " esm_F250          67   371\n",
      " esm_F984          67   372\n",
      " esm_F864          66   373\n",
      " esm_F457          66   374\n",
      " esm_F410          66   375\n",
      "esm_F1263          66   376\n",
      " esm_F115          66   377\n",
      " esm_F122          66   378\n",
      " esm_F734          66   379\n",
      " esm_F869          66   380\n",
      "esm_F1144          66   381\n",
      "esm_F1034          66   382\n",
      " esm_F762          66   383\n",
      "  esm_F29          66   384\n",
      "esm_F1067          66   385\n",
      " esm_F619          66   386\n",
      " esm_F220          66   387\n",
      "esm_F1134          66   388\n",
      "esm_F1133          66   389\n",
      " esm_F624          66   390\n",
      "esm_F1083          66   391\n",
      "esm_F1084          66   392\n",
      " esm_F632          66   393\n",
      "esm_F1100          66   394\n",
      " esm_F792          66   395\n",
      " esm_F583          66   396\n",
      " esm_F628          66   397\n",
      " esm_F919          66   398\n",
      " esm_F695          66   399\n",
      "  esm_F16          66   400\n",
      " esm_F324          66   401\n",
      " esm_F652          66   402\n",
      " esm_F596          66   403\n",
      " esm_F601          66   404\n",
      " esm_F956          66   405\n",
      " esm_F690          66   406\n",
      " esm_F598          66   407\n",
      "esm_F1066          65   408\n",
      " esm_F479          65   409\n",
      " esm_F562          65   410\n",
      "esm_F1138          65   411\n",
      " esm_F224          65   412\n",
      " esm_F548          65   413\n",
      " esm_F859          65   414\n",
      " esm_F832          65   415\n",
      "esm_F1268          65   416\n",
      " esm_F813          65   417\n",
      " esm_F633          65   418\n",
      " esm_F698          65   419\n",
      " esm_F149          65   420\n",
      "  esm_F67          65   421\n",
      "  esm_F13          65   422\n",
      " esm_F551          65   423\n",
      " esm_F370          65   424\n",
      "  esm_F36          65   425\n",
      " esm_F975          65   426\n",
      " esm_F797          65   427\n",
      "  esm_F78          65   428\n",
      " esm_F178          65   429\n",
      " esm_F215          65   430\n",
      " esm_F926          65   431\n",
      " esm_F488          65   432\n",
      "  esm_F33          64   433\n",
      " esm_F241          64   434\n",
      " esm_F242          64   435\n",
      " esm_F630          64   436\n",
      " esm_F362          64   437\n",
      "esm_F1137          64   438\n",
      " esm_F903          64   439\n",
      " esm_F462          64   440\n",
      " esm_F766          64   441\n",
      " esm_F836          64   442\n",
      " esm_F810          64   443\n",
      " esm_F151          64   444\n",
      " esm_F604          64   445\n",
      "esm_F1206          64   446\n",
      " esm_F827          64   447\n",
      "esm_F1072          64   448\n",
      " esm_F674          64   449\n",
      " esm_F532          64   450\n",
      " esm_F849          64   451\n",
      " esm_F202          64   452\n",
      " esm_F982          64   453\n",
      "esm_F1261          64   454\n",
      "esm_F1032          64   455\n",
      " esm_F853          64   456\n",
      "esm_F1005          64   457\n",
      " esm_F891          64   458\n",
      " esm_F272          64   459\n",
      " esm_F559          64   460\n",
      " esm_F268          64   461\n",
      " esm_F867          64   462\n",
      " esm_F117          64   463\n",
      "  esm_F98          64   464\n",
      "esm_F1031          64   465\n",
      " esm_F597          64   466\n",
      " esm_F186          64   467\n",
      " esm_F346          64   468\n",
      " esm_F312          64   469\n",
      " esm_F861          64   470\n",
      " esm_F705          64   471\n",
      "  esm_F79          64   472\n",
      " esm_F108          63   473\n",
      " esm_F670          63   474\n",
      " esm_F161          63   475\n",
      " esm_F491          63   476\n",
      " esm_F995          63   477\n",
      " esm_F210          63   478\n",
      " esm_F316          63   479\n",
      " esm_F977          63   480\n",
      "esm_F1117          63   481\n",
      " esm_F516          63   482\n",
      " esm_F993          63   483\n",
      " esm_F805          63   484\n",
      " esm_F360          63   485\n",
      "esm_F1205          63   486\n",
      " esm_F278          63   487\n",
      "esm_F1037          63   488\n",
      " esm_F134          63   489\n",
      " esm_F638          63   490\n",
      " esm_F635          63   491\n",
      " esm_F442          63   492\n",
      " esm_F544          63   493\n",
      " esm_F230          63   494\n",
      " esm_F761          63   495\n",
      "  esm_F21          63   496\n",
      " esm_F617          63   497\n",
      "esm_F1155          63   498\n",
      "esm_F1198          63   499\n",
      "  esm_F47          63   500\n",
      "esm_F1157          63   501\n",
      " esm_F329          63   502\n",
      " esm_F157          63   503\n",
      "esm_F1215          62   504\n",
      " esm_F603          62   505\n",
      " esm_F709          62   506\n",
      " esm_F593          62   507\n",
      " esm_F578          62   508\n",
      "  esm_F83          62   509\n",
      " esm_F555          62   510\n",
      " esm_F116          62   511\n",
      "  esm_F75          62   512\n",
      " esm_F325          62   513\n",
      "esm_F1103          62   514\n",
      " esm_F307          62   515\n",
      "esm_F1010          62   516\n",
      " esm_F257          62   517\n",
      " esm_F454          62   518\n",
      " esm_F291          62   519\n",
      " esm_F418          62   520\n",
      "esm_F1059          62   521\n",
      " esm_F469          62   522\n",
      " esm_F229          62   523\n",
      " esm_F822          62   524\n",
      " esm_F819          62   525\n",
      " esm_F816          62   526\n",
      " esm_F893          62   527\n",
      " esm_F300          62   528\n",
      " esm_F803          62   529\n",
      " esm_F492          62   530\n",
      " esm_F931          62   531\n",
      " esm_F928          62   532\n",
      " esm_F197          62   533\n",
      " esm_F374          62   534\n",
      " esm_F942          62   535\n",
      " esm_F299          61   536\n",
      " esm_F943          61   537\n",
      "esm_F1191          61   538\n",
      " esm_F305          61   539\n",
      "  esm_F22          61   540\n",
      " esm_F566          61   541\n",
      " esm_F988          61   542\n",
      "esm_F1158          61   543\n",
      "esm_F1182          61   544\n",
      "  esm_F20          61   545\n",
      "esm_F1165          61   546\n",
      " esm_F976          61   547\n",
      "esm_F1225          61   548\n",
      "   esm_F9          61   549\n",
      "esm_F1022          61   550\n",
      " esm_F343          61   551\n",
      " esm_F994          61   552\n",
      " esm_F413          61   553\n",
      " esm_F412          61   554\n",
      " esm_F974          61   555\n",
      " esm_F356          61   556\n",
      " esm_F428          61   557\n",
      " esm_F768          61   558\n",
      " esm_F841          61   559\n",
      " esm_F530          61   560\n",
      " esm_F534          61   561\n",
      "esm_F1127          61   562\n",
      "esm_F1026          61   563\n",
      " esm_F621          61   564\n",
      " esm_F818          61   565\n",
      " esm_F152          61   566\n",
      " esm_F221          61   567\n",
      "esm_F1243          61   568\n",
      " esm_F833          61   569\n",
      " esm_F150          61   570\n",
      " esm_F769          61   571\n",
      " esm_F240          61   572\n",
      "esm_F1259          61   573\n",
      " esm_F200          61   574\n",
      " esm_F444          61   575\n",
      "esm_F1097          61   576\n",
      "esm_F1250          61   577\n",
      "esm_F1080          60   578\n",
      "  esm_F88          60   579\n",
      " esm_F967          60   580\n",
      " esm_F350          60   581\n",
      "  esm_F40          60   582\n",
      " esm_F798          60   583\n",
      " esm_F902          60   584\n",
      "esm_F1116          60   585\n",
      " esm_F927          60   586\n",
      " esm_F389          60   587\n",
      " esm_F433          60   588\n",
      " esm_F219          60   589\n",
      " esm_F582          60   590\n",
      "esm_F1173          60   591\n",
      " esm_F874          60   592\n",
      "  esm_F55          60   593\n",
      "  esm_F53          60   594\n",
      " esm_F463          60   595\n",
      " esm_F996          60   596\n",
      " esm_F745          60   597\n",
      "  esm_F46          60   598\n",
      " esm_F287          60   599\n",
      " esm_F256          60   600\n",
      "esm_F1004          60   601\n",
      " esm_F748          60   602\n",
      " esm_F675          59   603\n",
      "  esm_F62          59   604\n",
      "esm_F1209          59   605\n",
      "esm_F1242          59   606\n",
      " esm_F658          59   607\n",
      " esm_F508          59   608\n",
      "esm_F1207          59   609\n",
      " esm_F512          59   610\n",
      " esm_F190          59   611\n",
      " esm_F234          59   612\n",
      "esm_F1044          59   613\n",
      " esm_F248          59   614\n",
      " esm_F258          59   615\n",
      " esm_F426          59   616\n",
      " esm_F999          59   617\n",
      " esm_F880          59   618\n",
      " esm_F887          59   619\n",
      " esm_F650          59   620\n",
      " esm_F962          59   621\n",
      " esm_F916          59   622\n",
      " esm_F358          59   623\n",
      " esm_F922          59   624\n",
      " esm_F925          59   625\n",
      " esm_F104          59   626\n",
      "esm_F1102          59   627\n",
      "esm_F1112          59   628\n",
      " esm_F177          59   629\n",
      " esm_F107          59   630\n",
      " esm_F711          59   631\n",
      "esm_F1168          59   632\n",
      " esm_F113          59   633\n",
      " esm_F563          59   634\n",
      " esm_F550          59   635\n",
      " esm_F756          59   636\n",
      " esm_F540          59   637\n",
      " esm_F163          59   638\n",
      " esm_F164          59   639\n",
      " esm_F170          59   640\n",
      " esm_F781          58   641\n",
      "esm_F1054          58   642\n",
      " esm_F782          58   643\n",
      "esm_F1156          58   644\n",
      "esm_F1061          58   645\n",
      " esm_F800          58   646\n",
      "esm_F1101          58   647\n",
      "esm_F1119          58   648\n",
      " esm_F937          58   649\n",
      "   esm_F1          58   650\n",
      " esm_F486          58   651\n",
      " esm_F218          58   652\n",
      " esm_F233          58   653\n",
      " esm_F334          58   654\n",
      " esm_F147          58   655\n",
      " esm_F467          58   656\n",
      " esm_F148          58   657\n",
      " esm_F447          58   658\n",
      " esm_F529          58   659\n",
      "  esm_F66          58   660\n",
      " esm_F476          58   661\n",
      " esm_F580          58   662\n",
      " esm_F581          58   663\n",
      " esm_F420          58   664\n",
      " esm_F456          58   665\n",
      "esm_F1033          57   666\n",
      " esm_F499          57   667\n",
      " esm_F826          57   668\n",
      "esm_F1049          57   669\n",
      "  esm_F91          57   670\n",
      " esm_F815          57   671\n",
      "esm_F1214          57   672\n",
      " esm_F332          57   673\n",
      "esm_F1018          57   674\n",
      " esm_F957          57   675\n",
      "esm_F1093          57   676\n",
      " esm_F908          57   677\n",
      " esm_F668          57   678\n",
      " esm_F206          57   679\n",
      " esm_F741          57   680\n",
      " esm_F907          57   681\n",
      " esm_F176          57   682\n",
      " esm_F561          57   683\n",
      " esm_F773          57   684\n",
      " esm_F537          57   685\n",
      "  esm_F42          57   686\n",
      " esm_F627          57   687\n",
      " esm_F536          57   688\n",
      " esm_F526          56   689\n",
      "esm_F1104          56   690\n",
      "esm_F1079          56   691\n",
      " esm_F804          56   692\n",
      "esm_F1143          56   693\n",
      "esm_F1111          56   694\n",
      "esm_F1019          56   695\n",
      " esm_F793          56   696\n",
      " esm_F500          56   697\n",
      " esm_F722          56   698\n",
      " esm_F264          56   699\n",
      " esm_F789          56   700\n",
      "esm_F1055          56   701\n",
      " esm_F270          56   702\n",
      " esm_F162          56   703\n",
      "esm_F1128          56   704\n",
      "esm_F1065          56   705\n",
      "  esm_F84          56   706\n",
      " esm_F665          56   707\n",
      " esm_F920          56   708\n",
      " esm_F847          56   709\n",
      " esm_F471          56   710\n",
      " esm_F686          56   711\n",
      " esm_F932          56   712\n",
      " esm_F983          56   713\n",
      " esm_F693          56   714\n",
      " esm_F481          56   715\n",
      " esm_F986          56   716\n",
      " esm_F482          56   717\n",
      " esm_F309          56   718\n",
      "  esm_F18          56   719\n",
      "esm_F1240          56   720\n",
      "  esm_F97          56   721\n",
      " esm_F406          56   722\n",
      " esm_F290          56   723\n",
      " esm_F440          56   724\n",
      " esm_F764          55   725\n",
      " esm_F288          55   726\n",
      " esm_F393          55   727\n",
      " esm_F438          55   728\n",
      " esm_F394          55   729\n",
      " esm_F929          55   730\n",
      " esm_F898          55   731\n",
      " esm_F399          55   732\n",
      " esm_F955          55   733\n",
      " esm_F450          55   734\n",
      " esm_F952          55   735\n",
      " esm_F953          55   736\n",
      " esm_F489          55   737\n",
      " esm_F349          55   738\n",
      " esm_F465          55   739\n",
      " esm_F979          55   740\n",
      " esm_F523          55   741\n",
      "esm_F1020          55   742\n",
      "esm_F1095          55   743\n",
      " esm_F297          55   744\n",
      " esm_F336          55   745\n",
      " esm_F424          55   746\n",
      "esm_F1159          55   747\n",
      "esm_F1255          55   748\n",
      " esm_F702          55   749\n",
      " esm_F735          55   750\n",
      "  esm_F57          55   751\n",
      " esm_F678          55   752\n",
      "esm_F1153          55   753\n",
      "  esm_F48          55   754\n",
      "  esm_F69          55   755\n",
      "  esm_F28          55   756\n",
      " esm_F715          55   757\n",
      "  esm_F23          55   758\n",
      "esm_F1162          55   759\n",
      "esm_F1274          55   760\n",
      " esm_F576          54   761\n",
      " esm_F799          54   762\n",
      " esm_F302          54   763\n",
      " esm_F495          54   764\n",
      " esm_F573          54   765\n",
      " esm_F276          54   766\n",
      "esm_F1096          54   767\n",
      " esm_F498          54   768\n",
      " esm_F196          54   769\n",
      "  esm_F70          54   770\n",
      " esm_F689          54   771\n",
      "  esm_F60          54   772\n",
      " esm_F509          54   773\n",
      " esm_F837          54   774\n",
      "  esm_F50          54   775\n",
      " esm_F938          54   776\n",
      " esm_F852          54   777\n",
      "esm_F1239          54   778\n",
      " esm_F863          54   779\n",
      " esm_F644          54   780\n",
      " esm_F645          54   781\n",
      " esm_F472          54   782\n",
      " esm_F225          54   783\n",
      " esm_F174          54   784\n",
      " esm_F199          54   785\n",
      " esm_F567          54   786\n",
      " esm_F128          54   787\n",
      " esm_F216          54   788\n",
      " esm_F779          54   789\n",
      " esm_F770          54   790\n",
      " esm_F524          54   791\n",
      " esm_F243          54   792\n",
      " esm_F751          54   793\n",
      " esm_F752          53   794\n",
      " esm_F464          53   795\n",
      " esm_F313          53   796\n",
      "esm_F1203          53   797\n",
      "esm_F1148          53   798\n",
      " esm_F729          53   799\n",
      " esm_F980          53   800\n",
      " esm_F828          53   801\n",
      "esm_F1141          53   802\n",
      "esm_F1042          53   803\n",
      "  esm_F15          53   804\n",
      " esm_F959          53   805\n",
      "esm_F1228          53   806\n",
      "  esm_F96          53   807\n",
      " esm_F187          53   808\n",
      "esm_F1082          53   809\n",
      " esm_F851          53   810\n",
      " esm_F667          53   811\n",
      " esm_F387          53   812\n",
      "esm_F1043          53   813\n",
      " esm_F320          53   814\n",
      " esm_F727          53   815\n",
      " esm_F584          53   816\n",
      " esm_F719          53   817\n",
      "esm_F1029          53   818\n",
      " esm_F594          53   819\n",
      "esm_F1170          53   820\n",
      " esm_F780          53   821\n",
      "esm_F1174          53   822\n",
      "esm_F1171          52   823\n",
      "esm_F1224          52   824\n",
      " esm_F123          52   825\n",
      " esm_F298          52   826\n",
      " esm_F616          52   827\n",
      "esm_F1048          52   828\n",
      " esm_F549          52   829\n",
      " esm_F950          52   830\n",
      " esm_F501          52   831\n",
      " esm_F855          52   832\n",
      "esm_F1027          52   833\n",
      " esm_F923          52   834\n",
      " esm_F552          52   835\n",
      "esm_F1106          52   836\n",
      " esm_F398          52   837\n",
      " esm_F341          52   838\n",
      "esm_F1007          52   839\n",
      " esm_F613          52   840\n",
      " esm_F589          52   841\n",
      " esm_F703          52   842\n",
      "esm_F1197          52   843\n",
      " esm_F528          52   844\n",
      " esm_F232          52   845\n",
      " esm_F825          52   846\n",
      " esm_F726          52   847\n",
      "esm_F1181          52   848\n",
      " esm_F228          52   849\n",
      "  esm_F54          52   850\n",
      " esm_F877          52   851\n",
      "  esm_F56          52   852\n",
      " esm_F330          51   853\n",
      "  esm_F25          51   854\n",
      " esm_F421          51   855\n",
      " esm_F188          51   856\n",
      " esm_F890          51   857\n",
      "esm_F1270          51   858\n",
      "esm_F1039          51   859\n",
      "esm_F1262          51   860\n",
      "  esm_F82          51   861\n",
      " esm_F263          51   862\n",
      "esm_F1199          51   863\n",
      "esm_F1167          51   864\n",
      " esm_F267          51   865\n",
      "  esm_F59          51   866\n",
      "esm_F1253          51   867\n",
      "esm_F1073          51   868\n",
      "esm_F1252          51   869\n",
      "esm_F1230          51   870\n",
      "  esm_F45          51   871\n",
      "esm_F1015          51   872\n",
      " esm_F207          51   873\n",
      "esm_F1179          51   874\n",
      "esm_F1110          51   875\n",
      " esm_F649          51   876\n",
      " esm_F620          51   877\n",
      " esm_F806          51   878\n",
      " esm_F546          51   879\n",
      " esm_F712          51   880\n",
      " esm_F846          51   881\n",
      " esm_F844          51   882\n",
      " esm_F474          51   883\n",
      " esm_F829          51   884\n",
      " esm_F483          51   885\n",
      " esm_F680          51   886\n",
      " esm_F812          51   887\n",
      " esm_F850          51   888\n",
      " esm_F676          51   889\n",
      " esm_F560          51   890\n",
      " esm_F660          51   891\n",
      " esm_F384          50   892\n",
      " esm_F142          50   893\n",
      " esm_F455          50   894\n",
      " esm_F132          50   895\n",
      " esm_F924          50   896\n",
      " esm_F732          50   897\n",
      " esm_F991          50   898\n",
      " esm_F622          50   899\n",
      " esm_F373          50   900\n",
      " esm_F913          50   901\n",
      " esm_F527          50   902\n",
      " esm_F453          50   903\n",
      "esm_F1152          50   904\n",
      "esm_F1235          50   905\n",
      " esm_F114          50   906\n",
      " esm_F365          50   907\n",
      " esm_F572          50   908\n",
      "esm_F1041          50   909\n",
      " esm_F392          50   910\n",
      " esm_F510          50   911\n",
      " esm_F466          50   912\n",
      " esm_F692          50   913\n",
      " esm_F326          50   914\n",
      " esm_F277          50   915\n",
      " esm_F736          50   916\n",
      " esm_F657          50   917\n",
      "esm_F1130          50   918\n",
      " esm_F901          50   919\n",
      " esm_F801          50   920\n",
      "esm_F1189          50   921\n",
      "esm_F1254          50   922\n",
      " esm_F340          50   923\n",
      "esm_F1213          50   924\n",
      "  esm_F30          50   925\n",
      " esm_F699          50   926\n",
      "esm_F1193          50   927\n",
      " esm_F821          49   928\n",
      " esm_F947          49   929\n",
      " esm_F286          49   930\n",
      " esm_F718          49   931\n",
      "  esm_F99          49   932\n",
      " esm_F262          49   933\n",
      " esm_F321          49   934\n",
      " esm_F473          49   935\n",
      " esm_F790          49   936\n",
      " esm_F354          49   937\n",
      "esm_F1021          49   938\n",
      " esm_F260          49   939\n",
      " esm_F266          49   940\n",
      " esm_F359          49   941\n",
      " esm_F717          49   942\n",
      " esm_F739          49   943\n",
      " esm_F361          49   944\n",
      " esm_F223          49   945\n",
      "esm_F1118          49   946\n",
      " esm_F172          49   947\n",
      " esm_F747          49   948\n",
      " esm_F886          49   949\n",
      "esm_F1089          49   950\n",
      " esm_F757          49   951\n",
      "esm_F1125          49   952\n",
      "esm_F1085          49   953\n",
      "esm_F1266          49   954\n",
      "esm_F1264          49   955\n",
      " esm_F246          49   956\n",
      "esm_F1257          49   957\n",
      " esm_F156          49   958\n",
      " esm_F244          49   959\n",
      "esm_F1234          49   960\n",
      " esm_F236          49   961\n",
      " esm_F934          49   962\n",
      " esm_F626          49   963\n",
      "esm_F1062          49   964\n",
      " esm_F133          49   965\n",
      "esm_F1057          49   966\n",
      " esm_F721          49   967\n",
      " esm_F518          49   968\n",
      "esm_F1249          49   969\n",
      "esm_F1009          48   970\n",
      " esm_F193          48   971\n",
      " esm_F738          48   972\n",
      "esm_F1006          48   973\n",
      " esm_F754          48   974\n",
      " esm_F811          48   975\n",
      " esm_F750          48   976\n",
      "esm_F1070          48   977\n",
      "esm_F1060          48   978\n",
      " esm_F506          48   979\n",
      " esm_F160          48   980\n",
      " esm_F102          48   981\n",
      " esm_F109          48   982\n",
      "esm_F1016          48   983\n",
      " esm_F422          48   984\n",
      "esm_F1068          48   985\n",
      " esm_F155          48   986\n",
      " esm_F154          48   987\n",
      "esm_F1074          48   988\n",
      " esm_F774          48   989\n",
      " esm_F760          48   990\n",
      " esm_F842          48   991\n",
      "  esm_F31          48   992\n",
      "  esm_F49          48   993\n",
      "  esm_F52          48   994\n",
      "esm_F1237          48   995\n",
      " esm_F380          48   996\n",
      " esm_F629          48   997\n",
      " esm_F838          48   998\n",
      "esm_F1247          48   999\n",
      "  esm_F73          48  1000\n",
      " esm_F910          48  1001\n",
      "esm_F1212          48  1002\n",
      "esm_F1265          48  1003\n",
      " esm_F655          48  1004\n",
      " esm_F414          48  1005\n",
      " esm_F314          48  1006\n",
      " esm_F946          48  1007\n",
      "esm_F1271          48  1008\n",
      " esm_F888          48  1009\n",
      " esm_F948          48  1010\n",
      " esm_F353          48  1011\n",
      "esm_F1233          47  1012\n",
      "esm_F1166          47  1013\n",
      " esm_F785          47  1014\n",
      "esm_F1195          47  1015\n",
      " esm_F367          47  1016\n",
      " esm_F930          47  1017\n",
      "esm_F1218          47  1018\n",
      " esm_F541          47  1019\n",
      " esm_F654          47  1020\n",
      " esm_F168          47  1021\n",
      " esm_F553          47  1022\n",
      " esm_F184          47  1023\n",
      " esm_F917          47  1024\n",
      "esm_F1149          47  1025\n",
      " esm_F602          47  1026\n",
      " esm_F961          47  1027\n",
      "  esm_F64          47  1028\n",
      " esm_F688          47  1029\n",
      " esm_F807          47  1030\n",
      " esm_F338          47  1031\n",
      " esm_F271          47  1032\n",
      "  esm_F92          47  1033\n",
      "esm_F1071          46  1034\n",
      " esm_F558          46  1035\n",
      " esm_F237          46  1036\n",
      " esm_F173          46  1037\n",
      " esm_F446          46  1038\n",
      " esm_F100          46  1039\n",
      "  esm_F24          46  1040\n",
      "esm_F1122          46  1041\n",
      " esm_F565          46  1042\n",
      " esm_F899          46  1043\n",
      "esm_F1277          46  1044\n",
      "   esm_F4          46  1045\n",
      " esm_F195          46  1046\n",
      "esm_F1135          46  1047\n",
      " esm_F280          46  1048\n",
      "esm_F1008          46  1049\n",
      " esm_F378          46  1050\n",
      " esm_F355          46  1051\n",
      " esm_F791          46  1052\n",
      " esm_F363          46  1053\n",
      "esm_F1227          46  1054\n",
      " esm_F796          46  1055\n",
      "  esm_F61          46  1056\n",
      " esm_F679          46  1057\n",
      "esm_F1025          46  1058\n",
      " esm_F125          46  1059\n",
      "esm_F1221          45  1060\n",
      "   esm_F7          45  1061\n",
      " esm_F900          45  1062\n",
      " esm_F824          45  1063\n",
      " esm_F167          45  1064\n",
      " esm_F868          45  1065\n",
      "esm_F1092          45  1066\n",
      " esm_F600          45  1067\n",
      " esm_F612          45  1068\n",
      " esm_F317          45  1069\n",
      " esm_F182          45  1070\n",
      " esm_F431          45  1071\n",
      " esm_F713          45  1072\n",
      " esm_F556          45  1073\n",
      " esm_F502          45  1074\n",
      " esm_F429          45  1075\n",
      " esm_F379          45  1076\n",
      "esm_F1147          45  1077\n",
      "esm_F1045          45  1078\n",
      " esm_F310          45  1079\n",
      " esm_F884          44  1080\n",
      " esm_F419          44  1081\n",
      " esm_F175          44  1082\n",
      " esm_F882          44  1083\n",
      "esm_F1190          44  1084\n",
      " esm_F753          44  1085\n",
      " esm_F192          44  1086\n",
      " esm_F112          44  1087\n",
      "  esm_F68          44  1088\n",
      " esm_F700          44  1089\n",
      "  esm_F43          44  1090\n",
      " esm_F964          44  1091\n",
      " esm_F723          44  1092\n",
      " esm_F599          44  1093\n",
      " esm_F239          44  1094\n",
      " esm_F972          44  1095\n",
      "esm_F1056          44  1096\n",
      " esm_F701          44  1097\n",
      " esm_F606          44  1098\n",
      "  esm_F41          44  1099\n",
      " esm_F666          44  1100\n",
      "esm_F1076          44  1101\n",
      "   esm_F2          44  1102\n",
      " esm_F484          44  1103\n",
      " esm_F138          43  1104\n",
      "esm_F1184          43  1105\n",
      " esm_F153          43  1106\n",
      "esm_F1150          43  1107\n",
      " esm_F637          43  1108\n",
      "esm_F1175          43  1109\n",
      "esm_F1160          43  1110\n",
      " esm_F423          43  1111\n",
      "esm_F1002          43  1112\n",
      "esm_F1077          43  1113\n",
      " esm_F871          43  1114\n",
      " esm_F485          43  1115\n",
      " esm_F809          43  1116\n",
      " esm_F808          43  1117\n",
      " esm_F507          43  1118\n",
      " esm_F245          43  1119\n",
      " esm_F382          43  1120\n",
      " esm_F385          43  1121\n",
      " esm_F912          43  1122\n",
      " esm_F531          43  1123\n",
      " esm_F909          43  1124\n",
      " esm_F372          43  1125\n",
      "esm_F1091          43  1126\n",
      " esm_F742          43  1127\n",
      "esm_F1177          42  1128\n",
      " esm_F129          42  1129\n",
      " esm_F445          42  1130\n",
      "esm_F1180          42  1131\n",
      " esm_F860          42  1132\n",
      " esm_F269          42  1133\n",
      " esm_F590          42  1134\n",
      " esm_F749          42  1135\n",
      " esm_F434          42  1136\n",
      " esm_F417          42  1137\n",
      " esm_F376          42  1138\n",
      " esm_F279          42  1139\n",
      " esm_F862          42  1140\n",
      "esm_F1226          42  1141\n",
      " esm_F981          42  1142\n",
      " esm_F605          42  1143\n",
      " esm_F461          42  1144\n",
      " esm_F663          42  1145\n",
      " esm_F569          41  1146\n",
      " esm_F677          41  1147\n",
      " esm_F369          41  1148\n",
      " esm_F118          41  1149\n",
      "esm_F1164          41  1150\n",
      " esm_F577          41  1151\n",
      " esm_F737          41  1152\n",
      " esm_F364          41  1153\n",
      "esm_F1099          41  1154\n",
      " esm_F941          41  1155\n",
      " esm_F997          41  1156\n",
      " esm_F535          41  1157\n",
      " esm_F817          41  1158\n",
      " esm_F181          41  1159\n",
      " esm_F906          41  1160\n",
      "esm_F1258          41  1161\n",
      " esm_F415          40  1162\n",
      " esm_F940          40  1163\n",
      " esm_F814          40  1164\n",
      "  esm_F89          40  1165\n",
      " esm_F105          40  1166\n",
      "esm_F1108          40  1167\n",
      " esm_F872          40  1168\n",
      "esm_F1030          40  1169\n",
      "esm_F1248          40  1170\n",
      " esm_F881          40  1171\n",
      " esm_F542          40  1172\n",
      "esm_F1094          40  1173\n",
      "  esm_F39          40  1174\n",
      " esm_F435          40  1175\n",
      " esm_F211          40  1176\n",
      " esm_F520          40  1177\n",
      " esm_F968          40  1178\n",
      " esm_F401          40  1179\n",
      " esm_F436          40  1180\n",
      " esm_F213          40  1181\n",
      "  esm_F72          39  1182\n",
      " esm_F294          39  1183\n",
      " esm_F437          39  1184\n",
      " esm_F609          39  1185\n",
      " esm_F611          39  1186\n",
      " esm_F318          39  1187\n",
      " esm_F227          39  1188\n",
      "esm_F1064          39  1189\n",
      " esm_F894          39  1190\n",
      " esm_F730          39  1191\n",
      "esm_F1279          39  1192\n",
      " esm_F653          39  1193\n",
      " esm_F777          39  1194\n",
      " esm_F939          38  1195\n",
      "esm_F1014          38  1196\n",
      " esm_F758          38  1197\n",
      "esm_F1035          38  1198\n",
      " esm_F694          38  1199\n",
      "esm_F1241          38  1200\n",
      "esm_F1109          38  1201\n",
      " esm_F146          38  1202\n",
      " esm_F285          38  1203\n",
      "  esm_F10          38  1204\n",
      " esm_F763          38  1205\n",
      " esm_F646          38  1206\n",
      " esm_F610          37  1207\n",
      " esm_F740          37  1208\n",
      " esm_F728          37  1209\n",
      " esm_F327          37  1210\n",
      "esm_F1145          37  1211\n",
      "esm_F1050          37  1212\n",
      " esm_F265          37  1213\n",
      " esm_F820          37  1214\n",
      " esm_F214          37  1215\n",
      " esm_F460          36  1216\n",
      " esm_F778          36  1217\n",
      " esm_F255          36  1218\n",
      " esm_F656          36  1219\n",
      "esm_F1132          36  1220\n",
      " esm_F490          36  1221\n",
      " esm_F103          36  1222\n",
      " esm_F304          36  1223\n",
      " esm_F400          36  1224\n",
      "esm_F1219          36  1225\n",
      " esm_F344          36  1226\n",
      "esm_F1202          36  1227\n",
      " esm_F691          36  1228\n",
      "esm_F1216          35  1229\n",
      "esm_F1058          35  1230\n",
      " esm_F897          35  1231\n",
      "esm_F1220          35  1232\n",
      " esm_F319          35  1233\n",
      " esm_F684          35  1234\n",
      " esm_F539          35  1235\n",
      " esm_F179          35  1236\n",
      " esm_F259          35  1237\n",
      " esm_F137          35  1238\n",
      " esm_F126          35  1239\n",
      " esm_F475          34  1240\n",
      " esm_F284          34  1241\n",
      " esm_F251          34  1242\n",
      " esm_F564          34  1243\n",
      " esm_F217          34  1244\n",
      " esm_F640          33  1245\n",
      " esm_F935          33  1246\n",
      "esm_F1051          33  1247\n",
      "esm_F1161          33  1248\n",
      " esm_F945          33  1249\n",
      " esm_F443          32  1250\n",
      "esm_F1267          32  1251\n",
      "esm_F1169          32  1252\n",
      " esm_F883          32  1253\n",
      "esm_F1183          32  1254\n",
      " esm_F348          32  1255\n",
      "esm_F1276          32  1256\n",
      "esm_F1201          32  1257\n",
      " esm_F895          31  1258\n",
      " esm_F408          31  1259\n",
      " esm_F987          31  1260\n",
      " esm_F554          31  1261\n",
      " esm_F840          31  1262\n",
      "esm_F1280          30  1263\n",
      " esm_F989          30  1264\n",
      "esm_F1278          30  1265\n",
      "esm_F1078          30  1266\n",
      "esm_F1210          30  1267\n",
      " esm_F235          30  1268\n",
      " esm_F636          29  1269\n",
      " esm_F592          29  1270\n",
      " esm_F856          28  1271\n",
      " esm_F121          27  1272\n",
      " esm_F971          27  1273\n",
      "esm_F1231          27  1274\n",
      "esm_F1121          27  1275\n",
      "esm_F1232          27  1276\n",
      " esm_F282          26  1277\n",
      " esm_F451          25  1278\n",
      "   esm_F8          22  1279\n",
      "esm_F1105          21  1280\n",
      "\n",
      "Saving the reordered dataset...\n",
      "Train saved: Train_SFbyLGBM_2560.csv\n",
      "Feature importance table saved: FeatureImportance_2560.csv\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>importance</th>\n",
       "      <th>rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>esm_F296</td>\n",
       "      <td>162</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>esm_F969</td>\n",
       "      <td>151</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>esm_F990</td>\n",
       "      <td>148</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>esm_F1140</td>\n",
       "      <td>135</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>esm_F1146</td>\n",
       "      <td>135</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1275</th>\n",
       "      <td>esm_F1232</td>\n",
       "      <td>27</td>\n",
       "      <td>1276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1276</th>\n",
       "      <td>esm_F282</td>\n",
       "      <td>26</td>\n",
       "      <td>1277</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1277</th>\n",
       "      <td>esm_F451</td>\n",
       "      <td>25</td>\n",
       "      <td>1278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1278</th>\n",
       "      <td>esm_F8</td>\n",
       "      <td>22</td>\n",
       "      <td>1279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1279</th>\n",
       "      <td>esm_F1105</td>\n",
       "      <td>21</td>\n",
       "      <td>1280</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1280 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        feature  importance  rank\n",
       "0      esm_F296         162     1\n",
       "1      esm_F969         151     2\n",
       "2      esm_F990         148     3\n",
       "3     esm_F1140         135     4\n",
       "4     esm_F1146         135     5\n",
       "...         ...         ...   ...\n",
       "1275  esm_F1232          27  1276\n",
       "1276   esm_F282          26  1277\n",
       "1277   esm_F451          25  1278\n",
       "1278     esm_F8          22  1279\n",
       "1279  esm_F1105          21  1280\n",
       "\n",
       "[1280 rows x 3 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_importance_rank_and_save(train_csv_path=\"./esm2_t33_650M_UR50D.Train.m7.csv\",\\\n",
    "                                 n_estimators=2560,\\\n",
    "                                 output_train_path=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5715b89-284c-4891-b752-903c7ac7143e",
   "metadata": {},
   "source": [
    "## Sort the test set according to the feature order arranged in the previous step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e3da86d7-de54-4be3-8d63-77496bd5861d",
   "metadata": {},
   "outputs": [],
   "source": [
    "TestData_ESM=pd.read_csv(\"./esm2_t33_650M_UR50D.Test.m7.csv\",header=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3299afeb-5981-4f33-b886-f75d63a17aba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1280\n"
     ]
    }
   ],
   "source": [
    "selected_Feat=pd.read_csv(\"./FeatureImportance_2560.csv\")\n",
    "selected_Feat.head()\n",
    "print(len(selected_Feat[\"feature\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a0403099-1b66-4a45-924d-f2b5f6662ec3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TestData_ESM.shape= (15211, 1282)\n",
      "TestData_ESM_afterSelectedByTrainData.shape= (15211, 1280)\n"
     ]
    }
   ],
   "source": [
    "TestData_ESM_afterSelectedByTrainData=TestData_ESM[selected_Feat[\"feature\"]]\n",
    "print(\"TestData_ESM.shape=\",TestData_ESM.shape)\n",
    "print(\"TestData_ESM_afterSelectedByTrainData.shape=\",TestData_ESM_afterSelectedByTrainData.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "80381d02-57b7-41ed-9b7f-774050d53763",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TestData_ESM.head()\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Class</th>\n",
       "      <th>esm_F1</th>\n",
       "      <th>esm_F2</th>\n",
       "      <th>esm_F3</th>\n",
       "      <th>esm_F4</th>\n",
       "      <th>esm_F5</th>\n",
       "      <th>esm_F6</th>\n",
       "      <th>esm_F7</th>\n",
       "      <th>esm_F8</th>\n",
       "      <th>...</th>\n",
       "      <th>esm_F1271</th>\n",
       "      <th>esm_F1272</th>\n",
       "      <th>esm_F1273</th>\n",
       "      <th>esm_F1274</th>\n",
       "      <th>esm_F1275</th>\n",
       "      <th>esm_F1276</th>\n",
       "      <th>esm_F1277</th>\n",
       "      <th>esm_F1278</th>\n",
       "      <th>esm_F1279</th>\n",
       "      <th>esm_F1280</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>neg_Ribosomal_gene=RPS8_entry=UPI000D223254_fi...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.045195</td>\n",
       "      <td>-0.026242</td>\n",
       "      <td>0.035307</td>\n",
       "      <td>-0.084170</td>\n",
       "      <td>-0.086876</td>\n",
       "      <td>-0.072543</td>\n",
       "      <td>0.039193</td>\n",
       "      <td>-0.111353</td>\n",
       "      <td>...</td>\n",
       "      <td>0.123329</td>\n",
       "      <td>-0.016147</td>\n",
       "      <td>-0.155198</td>\n",
       "      <td>0.084050</td>\n",
       "      <td>0.003115</td>\n",
       "      <td>0.039609</td>\n",
       "      <td>0.118738</td>\n",
       "      <td>-0.012175</td>\n",
       "      <td>-0.121088</td>\n",
       "      <td>0.034521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pos_CellAge_gene=SYK_entry=UPI0002276745_file=...</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.035823</td>\n",
       "      <td>-0.092163</td>\n",
       "      <td>-0.039488</td>\n",
       "      <td>0.045137</td>\n",
       "      <td>0.006262</td>\n",
       "      <td>0.012173</td>\n",
       "      <td>0.064685</td>\n",
       "      <td>-0.118377</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.005137</td>\n",
       "      <td>-0.052192</td>\n",
       "      <td>-0.052388</td>\n",
       "      <td>-0.045852</td>\n",
       "      <td>-0.034535</td>\n",
       "      <td>-0.094527</td>\n",
       "      <td>0.008995</td>\n",
       "      <td>-0.148214</td>\n",
       "      <td>0.004657</td>\n",
       "      <td>0.089250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>pos_inVivo_gene=PLAT_entry=UPI000B886BFD_file=...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.008305</td>\n",
       "      <td>0.030156</td>\n",
       "      <td>-0.011520</td>\n",
       "      <td>-0.027233</td>\n",
       "      <td>-0.041378</td>\n",
       "      <td>-0.001557</td>\n",
       "      <td>-0.004173</td>\n",
       "      <td>0.129016</td>\n",
       "      <td>...</td>\n",
       "      <td>0.141775</td>\n",
       "      <td>0.098776</td>\n",
       "      <td>0.048220</td>\n",
       "      <td>0.053019</td>\n",
       "      <td>-0.054822</td>\n",
       "      <td>0.137386</td>\n",
       "      <td>0.046485</td>\n",
       "      <td>-0.119550</td>\n",
       "      <td>-0.051403</td>\n",
       "      <td>0.040668</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>neg_Housekeeping_gene=TBP_entry=UPI001101CFD8_...</td>\n",
       "      <td>0</td>\n",
       "      <td>0.014527</td>\n",
       "      <td>-0.092227</td>\n",
       "      <td>-0.011226</td>\n",
       "      <td>-0.068805</td>\n",
       "      <td>0.172386</td>\n",
       "      <td>-0.001953</td>\n",
       "      <td>-0.005654</td>\n",
       "      <td>-0.067550</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.008305</td>\n",
       "      <td>-0.000077</td>\n",
       "      <td>-0.067812</td>\n",
       "      <td>0.022100</td>\n",
       "      <td>-0.001190</td>\n",
       "      <td>-0.166921</td>\n",
       "      <td>-0.077546</td>\n",
       "      <td>-0.104252</td>\n",
       "      <td>0.076800</td>\n",
       "      <td>0.054643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>pos_CellAge_gene=CKB_entry=UPI0006C5B195_file=...</td>\n",
       "      <td>1</td>\n",
       "      <td>0.095027</td>\n",
       "      <td>-0.023953</td>\n",
       "      <td>0.092681</td>\n",
       "      <td>-0.034241</td>\n",
       "      <td>-0.038278</td>\n",
       "      <td>-0.166392</td>\n",
       "      <td>0.028586</td>\n",
       "      <td>-0.032335</td>\n",
       "      <td>...</td>\n",
       "      <td>0.226402</td>\n",
       "      <td>-0.016719</td>\n",
       "      <td>0.005870</td>\n",
       "      <td>0.044361</td>\n",
       "      <td>-0.039911</td>\n",
       "      <td>-0.087283</td>\n",
       "      <td>0.140806</td>\n",
       "      <td>-0.167772</td>\n",
       "      <td>0.061509</td>\n",
       "      <td>0.061730</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1282 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  ID  Class    esm_F1  \\\n",
       "0  neg_Ribosomal_gene=RPS8_entry=UPI000D223254_fi...      0  0.045195   \n",
       "1  pos_CellAge_gene=SYK_entry=UPI0002276745_file=...      1 -0.035823   \n",
       "2  pos_inVivo_gene=PLAT_entry=UPI000B886BFD_file=...      1  0.008305   \n",
       "3  neg_Housekeeping_gene=TBP_entry=UPI001101CFD8_...      0  0.014527   \n",
       "4  pos_CellAge_gene=CKB_entry=UPI0006C5B195_file=...      1  0.095027   \n",
       "\n",
       "     esm_F2    esm_F3    esm_F4    esm_F5    esm_F6    esm_F7    esm_F8  ...  \\\n",
       "0 -0.026242  0.035307 -0.084170 -0.086876 -0.072543  0.039193 -0.111353  ...   \n",
       "1 -0.092163 -0.039488  0.045137  0.006262  0.012173  0.064685 -0.118377  ...   \n",
       "2  0.030156 -0.011520 -0.027233 -0.041378 -0.001557 -0.004173  0.129016  ...   \n",
       "3 -0.092227 -0.011226 -0.068805  0.172386 -0.001953 -0.005654 -0.067550  ...   \n",
       "4 -0.023953  0.092681 -0.034241 -0.038278 -0.166392  0.028586 -0.032335  ...   \n",
       "\n",
       "   esm_F1271  esm_F1272  esm_F1273  esm_F1274  esm_F1275  esm_F1276  \\\n",
       "0   0.123329  -0.016147  -0.155198   0.084050   0.003115   0.039609   \n",
       "1  -0.005137  -0.052192  -0.052388  -0.045852  -0.034535  -0.094527   \n",
       "2   0.141775   0.098776   0.048220   0.053019  -0.054822   0.137386   \n",
       "3  -0.008305  -0.000077  -0.067812   0.022100  -0.001190  -0.166921   \n",
       "4   0.226402  -0.016719   0.005870   0.044361  -0.039911  -0.087283   \n",
       "\n",
       "   esm_F1277  esm_F1278  esm_F1279  esm_F1280  \n",
       "0   0.118738  -0.012175  -0.121088   0.034521  \n",
       "1   0.008995  -0.148214   0.004657   0.089250  \n",
       "2   0.046485  -0.119550  -0.051403   0.040668  \n",
       "3  -0.077546  -0.104252   0.076800   0.054643  \n",
       "4   0.140806  -0.167772   0.061509   0.061730  \n",
       "\n",
       "[5 rows x 1282 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"TestData_ESM.head()\")\n",
    "TestData_ESM.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "05bd353a-adb2-45d7-8803-62310621f0fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TestData_ESM_afterSelectedByTrainData.head()\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>esm_F296</th>\n",
       "      <th>esm_F969</th>\n",
       "      <th>esm_F990</th>\n",
       "      <th>esm_F1140</th>\n",
       "      <th>esm_F1146</th>\n",
       "      <th>esm_F496</th>\n",
       "      <th>esm_F538</th>\n",
       "      <th>esm_F452</th>\n",
       "      <th>esm_F710</th>\n",
       "      <th>esm_F292</th>\n",
       "      <th>...</th>\n",
       "      <th>esm_F856</th>\n",
       "      <th>esm_F121</th>\n",
       "      <th>esm_F971</th>\n",
       "      <th>esm_F1231</th>\n",
       "      <th>esm_F1121</th>\n",
       "      <th>esm_F1232</th>\n",
       "      <th>esm_F282</th>\n",
       "      <th>esm_F451</th>\n",
       "      <th>esm_F8</th>\n",
       "      <th>esm_F1105</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.091077</td>\n",
       "      <td>0.023326</td>\n",
       "      <td>0.040699</td>\n",
       "      <td>-0.027704</td>\n",
       "      <td>-0.005654</td>\n",
       "      <td>0.101834</td>\n",
       "      <td>0.064425</td>\n",
       "      <td>0.042899</td>\n",
       "      <td>0.022347</td>\n",
       "      <td>0.003607</td>\n",
       "      <td>...</td>\n",
       "      <td>0.091954</td>\n",
       "      <td>-0.040525</td>\n",
       "      <td>-0.100042</td>\n",
       "      <td>0.109892</td>\n",
       "      <td>0.041105</td>\n",
       "      <td>-0.031484</td>\n",
       "      <td>-0.044984</td>\n",
       "      <td>0.057333</td>\n",
       "      <td>-0.111353</td>\n",
       "      <td>0.008600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.011296</td>\n",
       "      <td>0.068233</td>\n",
       "      <td>0.115895</td>\n",
       "      <td>0.106830</td>\n",
       "      <td>0.148152</td>\n",
       "      <td>0.083445</td>\n",
       "      <td>-0.014294</td>\n",
       "      <td>0.009641</td>\n",
       "      <td>0.056439</td>\n",
       "      <td>-0.057827</td>\n",
       "      <td>...</td>\n",
       "      <td>0.072799</td>\n",
       "      <td>-0.143573</td>\n",
       "      <td>-0.075477</td>\n",
       "      <td>-0.082581</td>\n",
       "      <td>0.037850</td>\n",
       "      <td>0.040991</td>\n",
       "      <td>0.062631</td>\n",
       "      <td>0.012295</td>\n",
       "      <td>-0.118377</td>\n",
       "      <td>-0.036205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.016804</td>\n",
       "      <td>-0.048595</td>\n",
       "      <td>0.075195</td>\n",
       "      <td>0.063373</td>\n",
       "      <td>0.095806</td>\n",
       "      <td>0.083714</td>\n",
       "      <td>-0.063318</td>\n",
       "      <td>-0.058878</td>\n",
       "      <td>0.100478</td>\n",
       "      <td>0.015682</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.070193</td>\n",
       "      <td>0.021058</td>\n",
       "      <td>0.019608</td>\n",
       "      <td>0.120808</td>\n",
       "      <td>0.007931</td>\n",
       "      <td>0.142935</td>\n",
       "      <td>-0.064287</td>\n",
       "      <td>-0.001759</td>\n",
       "      <td>0.129016</td>\n",
       "      <td>0.046061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.061942</td>\n",
       "      <td>0.016521</td>\n",
       "      <td>0.207001</td>\n",
       "      <td>0.224142</td>\n",
       "      <td>0.054998</td>\n",
       "      <td>-0.100175</td>\n",
       "      <td>0.055636</td>\n",
       "      <td>0.041060</td>\n",
       "      <td>0.039134</td>\n",
       "      <td>-0.022436</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.009799</td>\n",
       "      <td>-0.161369</td>\n",
       "      <td>0.014544</td>\n",
       "      <td>0.026045</td>\n",
       "      <td>-0.034989</td>\n",
       "      <td>0.039334</td>\n",
       "      <td>0.019354</td>\n",
       "      <td>-0.004843</td>\n",
       "      <td>-0.067550</td>\n",
       "      <td>-0.026756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.078873</td>\n",
       "      <td>0.066595</td>\n",
       "      <td>0.037792</td>\n",
       "      <td>-0.039074</td>\n",
       "      <td>0.007160</td>\n",
       "      <td>-0.056257</td>\n",
       "      <td>-0.051296</td>\n",
       "      <td>0.001098</td>\n",
       "      <td>0.082934</td>\n",
       "      <td>-0.030642</td>\n",
       "      <td>...</td>\n",
       "      <td>0.068867</td>\n",
       "      <td>-0.080200</td>\n",
       "      <td>-0.077682</td>\n",
       "      <td>0.004961</td>\n",
       "      <td>-0.034283</td>\n",
       "      <td>0.018258</td>\n",
       "      <td>-0.024681</td>\n",
       "      <td>0.018788</td>\n",
       "      <td>-0.032335</td>\n",
       "      <td>-0.060810</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1280 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   esm_F296  esm_F969  esm_F990  esm_F1140  esm_F1146  esm_F496  esm_F538  \\\n",
       "0 -0.091077  0.023326  0.040699  -0.027704  -0.005654  0.101834  0.064425   \n",
       "1 -0.011296  0.068233  0.115895   0.106830   0.148152  0.083445 -0.014294   \n",
       "2  0.016804 -0.048595  0.075195   0.063373   0.095806  0.083714 -0.063318   \n",
       "3  0.061942  0.016521  0.207001   0.224142   0.054998 -0.100175  0.055636   \n",
       "4  0.078873  0.066595  0.037792  -0.039074   0.007160 -0.056257 -0.051296   \n",
       "\n",
       "   esm_F452  esm_F710  esm_F292  ...  esm_F856  esm_F121  esm_F971  esm_F1231  \\\n",
       "0  0.042899  0.022347  0.003607  ...  0.091954 -0.040525 -0.100042   0.109892   \n",
       "1  0.009641  0.056439 -0.057827  ...  0.072799 -0.143573 -0.075477  -0.082581   \n",
       "2 -0.058878  0.100478  0.015682  ... -0.070193  0.021058  0.019608   0.120808   \n",
       "3  0.041060  0.039134 -0.022436  ... -0.009799 -0.161369  0.014544   0.026045   \n",
       "4  0.001098  0.082934 -0.030642  ...  0.068867 -0.080200 -0.077682   0.004961   \n",
       "\n",
       "   esm_F1121  esm_F1232  esm_F282  esm_F451    esm_F8  esm_F1105  \n",
       "0   0.041105  -0.031484 -0.044984  0.057333 -0.111353   0.008600  \n",
       "1   0.037850   0.040991  0.062631  0.012295 -0.118377  -0.036205  \n",
       "2   0.007931   0.142935 -0.064287 -0.001759  0.129016   0.046061  \n",
       "3  -0.034989   0.039334  0.019354 -0.004843 -0.067550  -0.026756  \n",
       "4  -0.034283   0.018258 -0.024681  0.018788 -0.032335  -0.060810  \n",
       "\n",
       "[5 rows x 1280 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"TestData_ESM_afterSelectedByTrainData.head()\")\n",
    "TestData_ESM_afterSelectedByTrainData.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b8890231-21b3-422c-b66c-3a9a9d2d2206",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15211, 1282)\n",
      "                                                  ID  Class  esm_F296  \\\n",
      "0  neg_Ribosomal_gene=RPS8_entry=UPI000D223254_fi...      0 -0.091077   \n",
      "1  pos_CellAge_gene=SYK_entry=UPI0002276745_file=...      1 -0.011296   \n",
      "2  pos_inVivo_gene=PLAT_entry=UPI000B886BFD_file=...      1  0.016804   \n",
      "3  neg_Housekeeping_gene=TBP_entry=UPI001101CFD8_...      0  0.061942   \n",
      "4  pos_CellAge_gene=CKB_entry=UPI0006C5B195_file=...      1  0.078873   \n",
      "\n",
      "   esm_F969  esm_F990  esm_F1140  esm_F1146  esm_F496  esm_F538  esm_F452  \\\n",
      "0  0.023326  0.040699  -0.027704  -0.005654  0.101834  0.064425  0.042899   \n",
      "1  0.068233  0.115895   0.106830   0.148152  0.083445 -0.014294  0.009641   \n",
      "2 -0.048595  0.075195   0.063373   0.095806  0.083714 -0.063318 -0.058878   \n",
      "3  0.016521  0.207001   0.224142   0.054998 -0.100175  0.055636  0.041060   \n",
      "4  0.066595  0.037792  -0.039074   0.007160 -0.056257 -0.051296  0.001098   \n",
      "\n",
      "   ...  esm_F856  esm_F121  esm_F971  esm_F1231  esm_F1121  esm_F1232  \\\n",
      "0  ...  0.091954 -0.040525 -0.100042   0.109892   0.041105  -0.031484   \n",
      "1  ...  0.072799 -0.143573 -0.075477  -0.082581   0.037850   0.040991   \n",
      "2  ... -0.070193  0.021058  0.019608   0.120808   0.007931   0.142935   \n",
      "3  ... -0.009799 -0.161369  0.014544   0.026045  -0.034989   0.039334   \n",
      "4  ...  0.068867 -0.080200 -0.077682   0.004961  -0.034283   0.018258   \n",
      "\n",
      "   esm_F282  esm_F451    esm_F8  esm_F1105  \n",
      "0 -0.044984  0.057333 -0.111353   0.008600  \n",
      "1  0.062631  0.012295 -0.118377  -0.036205  \n",
      "2 -0.064287 -0.001759  0.129016   0.046061  \n",
      "3  0.019354 -0.004843 -0.067550  -0.026756  \n",
      "4 -0.024681  0.018788 -0.032335  -0.060810  \n",
      "\n",
      "[5 rows x 1282 columns]\n"
     ]
    }
   ],
   "source": [
    "TestData_ESM_afterSelectedByTrainDataWithLabel=pd.concat([TestData_ESM.iloc[:,:2],\\\n",
    "                                                          TestData_ESM_afterSelectedByTrainData],axis=1)\n",
    "print(TestData_ESM_afterSelectedByTrainDataWithLabel.shape)\n",
    "print(TestData_ESM_afterSelectedByTrainDataWithLabel.head())\n",
    "TestData_ESM_afterSelectedByTrainDataWithLabel.to_csv(\"./TestData_ESM_afterSelectedByTrainDataWithLabel.csv\",index=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30aa9c19-4ea3-4e06-9b4d-bb9118fc1cbf",
   "metadata": {},
   "source": [
    "# Five-Fold Cross validation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a528905-ae8a-4898-9ac0-f74e6a2ce1f0",
   "metadata": {},
   "source": [
    "For cross-validation, we only do five-fold non-overlapping splits on the training set."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4cb3dc64-c1b4-445b-9f49-0b35dc012667",
   "metadata": {},
   "source": [
    "## Training Dataset Disjoint Splits"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa8cf273-52c4-44b8-bc46-a00358cf67c5",
   "metadata": {},
   "source": [
    "### SPLIT CONFIGURATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "533c234f-c1c8-4840-a492-8e8d6f33dad0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✔ config ready\n",
      "  split CSV : splits/five_fold.csv\n",
      "  reuse     : True\n"
     ]
    }
   ],
   "source": [
    "# ==================================================================\n",
    "# Cell 3.1.1: SPLIT CONFIGURATION\n",
    "# - Single source of truth for paths, seed and fold count\n",
    "# - Safe to rerun: existing outputs are NEVER overwritten blindly\n",
    "# ==================================================================\n",
    "\n",
    "from pathlib import Path\n",
    "import numpy as np\n",
    "\n",
    "SEED = 42\n",
    "N_FOLDS = 5\n",
    "CHUNKSIZE = 200_000\n",
    "\n",
    "DATA_PATH = \"./Train_SFbyLGBM_2560.csv\"\n",
    "SPLIT_DIR = Path(\"./splits\")\n",
    "SPLIT_CSV = SPLIT_DIR / \"five_fold.csv\"\n",
    "META_PATH = SPLIT_DIR / \"five_fold.meta.txt\"\n",
    "\n",
    "SPLIT_DIR.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "# Re-run protection: set to False if you want a fresh split\n",
    "REUSE_IF_EXISTS = True\n",
    "\n",
    "assert Path(DATA_PATH).exists(), f\"DATA_PATH missing: {DATA_PATH}\"\n",
    "print(\"✔ config ready\")\n",
    "print(\"  split CSV :\", SPLIT_CSV)\n",
    "print(\"  reuse     :\", REUSE_IF_EXISTS)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ef15604-2bd8-4e5e-91c4-2df11c938bc0",
   "metadata": {},
   "source": [
    "### Stream IDs and count samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1d7a8034-c268-4845-8017-b1122b96b1af",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">> Reusing existing split; skip reading raw data.\n"
     ]
    }
   ],
   "source": [
    "# ==================================================================\n",
    "# Cell 3.1.2: COUNT SAMPLES (ID column only)\n",
    "# - Features/labels are NEVER loaded here\n",
    "# - Result is a plain Series of IDs, one row per sample\n",
    "# ==================================================================\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "def stream_ids(path: str, chunksize: int = CHUNKSIZE) -> pd.Series:\n",
    "    \"\"\"Yield the ID column in chunks (memory-friendly).\"\"\"\n",
    "    reader = pd.read_csv(\n",
    "        path,\n",
    "        usecols=[0],      # first column = ID\n",
    "        dtype=str,\n",
    "        chunksize=chunksize,\n",
    "        iterator=True,\n",
    "    )\n",
    "    for chunk in reader:\n",
    "        yield chunk.iloc[:, 0].str.strip()\n",
    "\n",
    "if REUSE_IF_EXISTS and SPLIT_CSV.exists():\n",
    "    print(\">> Reusing existing split; skip reading raw data.\")\n",
    "    n_total = None\n",
    "else:\n",
    "    id_parts = []\n",
    "    for ids in stream_ids(DATA_PATH):\n",
    "        id_parts.append(ids)\n",
    "\n",
    "    all_ids = pd.concat(id_parts, ignore_index=True)\n",
    "    n_total = len(all_ids)\n",
    "\n",
    "    n_dup = int(all_ids.duplicated().sum())\n",
    "    print(f\"    total samples : {n_total}\")\n",
    "    print(f\"    duplicate IDs : {n_dup}\")\n",
    "    assert n_dup == 0, \"Duplicate IDs detected. Aborting.\"\n",
    "\n",
    "    # keep `all_ids` in memory for Cell 3\n",
    "    _all_ids = all_ids"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12eb38cd-ef20-40d5-84ae-2b05c467291f",
   "metadata": {},
   "source": [
    "### Deterministic 5-fold assignment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c914609b-f5c2-400a-ab97-d9a6d0b3ff22",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">> Split already exists; skipping generation.\n"
     ]
    }
   ],
   "source": [
    "# ==================================================================\n",
    "# Cell 3.1.3: DETERMINISTIC ID-LEVEL 5-FOLD ASSIGNMENT\n",
    "# - One reproducible permutation of IDs\n",
    "# - fold ∈ {0..K-1}; every ID appears in exactly one val set\n",
    "# ==================================================================\n",
    "\n",
    "if REUSE_IF_EXISTS and SPLIT_CSV.exists():\n",
    "    print(\">> Split already exists; skipping generation.\")\n",
    "    split_df = None  # will be loaded cleanly in Cell 5\n",
    "else:\n",
    "    rng = np.random.default_rng(SEED)\n",
    "    order = rng.permutation(n_total)\n",
    "\n",
    "    folds = np.empty(n_total, dtype=np.int8)\n",
    "    for k in range(N_FOLDS):\n",
    "        folds[order % N_FOLDS == k] = k\n",
    "\n",
    "    split_df = pd.DataFrame({\n",
    "        \"ID\": _all_ids.values,\n",
    "        \"fold\": folds,\n",
    "    })\n",
    "\n",
    "    # ---- structural invariants ----\n",
    "    assert len(split_df) == n_total\n",
    "    assert split_df[\"ID\"].is_unique\n",
    "    assert split_df[\"fold\"].nunique() == N_FOLDS\n",
    "\n",
    "    print(\"✔ fold distribution:\")\n",
    "    print(split_df[\"fold\"].value_counts().sort_index().to_string())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "decf10cb-a054-4bd3-a0b0-82e5539bc032",
   "metadata": {},
   "source": [
    "### Accounting + leakage audit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c3d262e6-793e-4cfb-8bbe-3874d4e9ceaa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ==================================================================\n",
    "# Cell 3.1.4: SPLIT AUDIT (per-fold + global + cross-fold)\n",
    "# - val + train == n_total       for every fold\n",
    "# - sum(val_i)   == n_total\n",
    "# - sum(train_i) == n_total*(K-1)\n",
    "# - val ∩ train  == ∅            within and across folds\n",
    "# ==================================================================\n",
    "\n",
    "# Only audit a freshly built split; reloaded files are audited in Cell 5\n",
    "if split_df is not None:\n",
    "\n",
    "    print(f\"{'fold':>5} {'val':>10} {'train':>10} {'val+train':>12} {'ok':>6}\")\n",
    "    print(\"-\" * 48)\n",
    "\n",
    "    val_counts, train_counts = [], []\n",
    "\n",
    "    for k in range(N_FOLDS):\n",
    "        val_mask = split_df[\"fold\"] == k\n",
    "        train_mask = ~val_mask\n",
    "\n",
    "        n_val = int(val_mask.sum())\n",
    "        n_train = int(train_mask.sum())\n",
    "        val_counts.append(n_val)\n",
    "        train_counts.append(n_train)\n",
    "\n",
    "        ok = (n_val + n_train) == len(split_df)\n",
    "        print(f\"{k:>5} {n_val:>10} {n_train:>10} {n_val + n_train:>12} {'✔' if ok else '✘'}\")\n",
    "\n",
    "        assert ok, f\"Fold {k}: row count mismatch\"\n",
    "\n",
    "        val_ids = split_df.loc[val_mask, \"ID\"]\n",
    "        train_ids = split_df.loc[train_mask, \"ID\"]\n",
    "\n",
    "        assert val_ids.is_unique and train_ids.is_unique\n",
    "        assert set(val_ids.to_numpy()).isdisjoint(set(train_ids.to_numpy())), \\\n",
    "            f\"Leakage in fold {k}\"\n",
    "\n",
    "    # ---- global accounting ----\n",
    "    sum_val = sum(val_counts)\n",
    "    sum_train = sum(train_counts)\n",
    "\n",
    "    print(\"\\n>> global accounting\")\n",
    "    print(f\"   Σ(val)   = {sum_val}  (expect {len(split_df)})\")\n",
    "    print(f\"   Σ(train) = {sum_train}  (expect {len(split_df)*(N_FOLDS-1)})\")\n",
    "\n",
    "    assert sum_val == len(split_df)\n",
    "    assert sum_train == len(split_df) * (N_FOLDS - 1)\n",
    "\n",
    "    # ---- cross-fold val disjointness ----\n",
    "    for i in range(N_FOLDS):\n",
    "        a = set(split_df.loc[split_df[\"fold\"] == i, \"ID\"].to_numpy())\n",
    "        for j in range(i + 1, N_FOLDS):\n",
    "            b = set(split_df.loc[split_df[\"fold\"] == j, \"ID\"].to_numpy())\n",
    "            assert a.isdisjoint(b), f\"fold {i} ∩ fold {j} overlap\"\n",
    "\n",
    "    print(\"\\n✔ ALL SPLIT INVARIANTS PASSED\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2477162-4b2a-4ebf-b12e-bb6fa43ae709",
   "metadata": {},
   "source": [
    "## CrossValidation and Test"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5ca3672d-6ba7-4234-9f2c-cb0fd32b4cfd",
   "metadata": {},
   "source": [
    "### Persist + reload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "339e7e58-a53f-4ab8-81ee-9cc8c8db2267",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✔ authoritative split loaded: splits/five_fold.csv\n",
      "  rows : 60843\n",
      "  folds: [0, 1, 2, 3, 4]\n"
     ]
    }
   ],
   "source": [
    "# ==================================================================\n",
    "# Cell 3.2.1: PERSIST SPLIT AND RELOAD (authoritative loader)\n",
    "# - Writes exactly one CSV (no duplicates, no raw data, no labels)\n",
    "# - Reloads it and re-runs the invariants as a contract\n",
    "# - Exposes `load_split()` for every downstream notebook\n",
    "# ==================================================================\n",
    "\n",
    "def load_split(path: str | Path = SPLIT_CSV,\n",
    "               n_folds: int = N_FOLDS) -> pd.DataFrame:\n",
    "    \"\"\"\n",
    "    Load the authoritative ID->fold mapping.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    DataFrame with columns [ID, fold]\n",
    "    ID is unique, fold ∈ {0..n_folds-1}\n",
    "    \"\"\"\n",
    "    p = Path(path)\n",
    "    assert p.exists(), f\"Split file missing: {p}\"\n",
    "\n",
    "    df = pd.read_csv(p, dtype={\"ID\": str})\n",
    "    assert list(df.columns)[:2] == [\"ID\", \"fold\"], df.columns.tolist()\n",
    "    assert df[\"ID\"].is_unique, \"ID collision after reload\"\n",
    "    assert df[\"fold\"].between(0, n_folds - 1).all(), \"invalid fold value\"\n",
    "    assert df[\"fold\"].nunique() == n_folds\n",
    "    return df\n",
    "\n",
    "\n",
    "if split_df is not None:\n",
    "    # ---- write authoritative file ----\n",
    "    SPLIT_CSV.write_text(\n",
    "        split_df.to_csv(index=False),\n",
    "        encoding=\"utf-8\",\n",
    "    )\n",
    "\n",
    "    # ---- write human-readable audit meta ----\n",
    "    lines = [\n",
    "        f\"data       : {DATA_PATH}\",\n",
    "        f\"seed       : {SEED}\",\n",
    "        f\"n_folds    : {N_FOLDS}\",\n",
    "        f\"n_samples  : {len(split_df)}\",\n",
    "        \"columns    : ID,fold\",\n",
    "        \"contract   : sum(val)=n_samples; sum(train)=n_samples*(K-1); val∩train=∅\",\n",
    "    ]\n",
    "    for k in range(N_FOLDS):\n",
    "        lines.append(f\"fold {k} val : {(split_df['fold']==k).sum()}\")\n",
    "    META_PATH.write_text(\"\\n\".join(lines) + \"\\n\", encoding=\"utf-8\")\n",
    "\n",
    "# ---- contract: always load from disk, never from a stale variable ----\n",
    "split_df = load_split()\n",
    "print(\"✔ authoritative split loaded:\", SPLIT_CSV)\n",
    "print(\"  rows :\", len(split_df))\n",
    "print(\"  folds:\", sorted(split_df[\"fold\"].unique()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8d977e8-f9b9-4a0a-8dd6-184c995bb08a",
   "metadata": {},
   "source": [
    "### END-TO-END CONSISTENCY CHECK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "18558667-1971-40fe-9bf5-fa743199d905",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✔ raw-file ID universe matches split exactly\n",
      "   raw unique IDs : 60843\n",
      "   split rows     : 60843\n"
     ]
    }
   ],
   "source": [
    "# ==================================================================\n",
    "# Cell 3.2.2: END-TO-END CONSISTENCY CHECK (optional but recommended)\n",
    "# - Streams the raw file again and groups by fold\n",
    "# - Proves the split covers exactly the same ID universe\n",
    "# ==================================================================\n",
    "\n",
    "seen_ids, counts = set(), {k: 0 for k in range(N_FOLDS)}\n",
    "\n",
    "for ids in stream_ids(DATA_PATH):\n",
    "    for i in range(len(ids)):\n",
    "        id_val = ids.iat[i]\n",
    "        seen_ids.add(id_val)\n",
    "        # we only need the set cardinality; fold lookup is cheap\n",
    "counts_total = len(seen_ids)\n",
    "\n",
    "assert seen_ids == set(split_df[\"ID\"])\n",
    "assert counts_total == len(split_df)\n",
    "\n",
    "print(\"✔ raw-file ID universe matches split exactly\")\n",
    "print(\"   raw unique IDs :\", counts_total)\n",
    "print(\"   split rows     :\", len(split_df))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bca07f0b-64fb-4953-8f79-bba1bce6375d",
   "metadata": {},
   "source": [
    "### Config & paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "4133f003-61ec-47a2-bd9c-8860a1be088b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✔ Config OK\n",
      "  split : ./splits/five_fold.csv\n",
      "  top_k : 450\n"
     ]
    }
   ],
   "source": [
    "# ==================================================================\n",
    "# Cell 3.2.3: CONFIGURATION\n",
    "# - All paths, hyperparameters and output locations in one place\n",
    "# - Fails fast if any required file is missing\n",
    "# ==================================================================\n",
    "\n",
    "from pathlib import Path\n",
    "import numpy as np\n",
    "\n",
    "SEED = 42\n",
    "N_FOLDS = 5\n",
    "TOP_K = 450 # e.g. 500 → top-500 features; None → all ordered features\n",
    "cv_run_done = False  \n",
    "DATA_TRAIN = \"./Train_SFbyLGBM_2560.csv\"\n",
    "DATA_TEST  = \"./TestData_ESM_afterSelectedByTrainDataWithLabel.csv\"\n",
    "SPLIT_CSV  = \"./splits/five_fold.csv\"\n",
    "FEATURE_ORDER_PATH = \"./FeatureImportance_2560.csv\"\n",
    "\n",
    "OUT_DIR   = Path(\"./five-fold-results\")\n",
    "MODEL_DIR = OUT_DIR / \"models\"\n",
    "OUT_DIR.mkdir(parents=True, exist_ok=True)\n",
    "MODEL_DIR.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "# SVM settings (change here only)\n",
    "SVM_C =  1.895682162\t\n",
    "SVM_GAMMA = 0.002739241\n",
    "SVM_KERNEL=\"rbf\"\n",
    "SVM_CLASS_WEIGHT = \"balanced\"\n",
    "\n",
    "# ---- sanity checks ----\n",
    "for f in [DATA_TRAIN, DATA_TEST, SPLIT_CSV, FEATURE_ORDER_PATH]:\n",
    "    assert Path(f).exists(), f\"MISSING FILE: {f}\"\n",
    "\n",
    "print(\"✔ Config OK\")\n",
    "print(\"  split :\", SPLIT_CSV)\n",
    "print(\"  top_k :\", TOP_K)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "24da17db-7b38-4f69-8796-9e224046307d",
   "metadata": {},
   "source": [
    "### Load split under the same contract"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "fb83854c-dd7f-496a-8395-f3345ef4113e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✔ Split loaded\n",
      "  rows : 60843\n",
      "  folds: [0, 1, 2, 3, 4]\n"
     ]
    }
   ],
   "source": [
    "# ==================================================================\n",
    "# Cell 3.2.4: LOAD SPLIT (same contract as build_splits.py)\n",
    "# - ID -> fold only, no features, no labels\n",
    "# - Guaranteed: unique ID, folds ∈ {0..K-1}, K folds present\n",
    "# ==================================================================\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "def load_split(path=SPLIT_CSV, n_folds=N_FOLDS) -> pd.DataFrame:\n",
    "    df = pd.read_csv(path, dtype={\"ID\": str})\n",
    "    assert list(df.columns)[:2] == [\"ID\", \"fold\"], df.columns.tolist()\n",
    "    assert df[\"ID\"].is_unique, \"ID collision in split file\"\n",
    "    assert df[\"fold\"].between(0, n_folds - 1).all(), \"invalid fold value\"\n",
    "    assert df[\"fold\"].nunique() == n_folds\n",
    "    return df\n",
    "\n",
    "split_df = load_split()\n",
    "print(\"✔ Split loaded\")\n",
    "print(\"  rows :\", len(split_df))\n",
    "print(\"  folds:\", sorted(split_df[\"fold\"].unique()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e9ea97f-bd6a-4867-b8d4-d9b38cda96b8",
   "metadata": {},
   "source": [
    "### Feature order + streaming data tools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "41db099c-98dc-40b7-8a17-822796fd0969",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==============================================================\n",
      "SCHEMA (position-based)\n",
      "==============================================================\n",
      "  train ID col     : 'ID'\n",
      "  train label col  : 'Class'\n",
      "  test  ID col     : 'ID'\n",
      "  test  label col  : 'Class'\n",
      "  data features    : 1280\n",
      "  order-file names : 450\n",
      "  top_k selected   : 450\n",
      "\n",
      "FEATURE CONSISTENCY\n",
      "  order ∩ data     : 450\n",
      "  in order not data: 0\n",
      "  in data not order: 830\n",
      "\n",
      "⚠ 830 feature(s) in the CSV are not ranked (they will be ignored): e.g. ['esm_F79', 'esm_F476', 'esm_F350', 'esm_F1093', 'esm_F620']\n",
      "  ranked == first 450 CSV cols? True\n",
      "\n",
      "FIRST 5 SELECTED FEATURES\n",
      "  1. esm_F296\n",
      "  2. esm_F969\n",
      "  3. esm_F990\n",
      "  4. esm_F1140\n",
      "  5. esm_F1146\n",
      "\n",
      "✔ Cell T2 OK: all ranked features are present in the data.\n"
     ]
    }
   ],
   "source": [
    "# ==================================================================\n",
    "# Cell 3.2.5: FEATURE ORDER & DATA LOADING UTILITIES\n",
    "# ==================================================================\n",
    "#\n",
    "# Dataset schema is defined BY POSITION, never by hard-coded names:\n",
    "#   column[0] = sample ID   (train: \"Unnamed: 0\", test: \"ID\")\n",
    "#   column[1] = label       (\"Class\" or \"label\", values 0/1)\n",
    "#   column[2:] = features   (e.g. esm_F296, esm_F969, ...)\n",
    "#\n",
    "# Feature-order file format (row 2+):\n",
    "#   A) plain name       : esm_F296\n",
    "#   B) rank / score file: esm_F296,162,1\n",
    "#       -> ONLY the first field before the separator is used\n",
    "#       -> file is assumed pre-sorted; top_k simply truncates\n",
    "#\n",
    "# ==================================================================\n",
    "\n",
    "from pathlib import Path\n",
    "import pandas as pd\n",
    "\n",
    "# ------------------------- Position contract ------------------------- #\n",
    "ID_POS = 0       # 0-based index of the ID column\n",
    "LABEL_POS = 1    # 0-based index of the label column\n",
    "\n",
    "\n",
    "# ------------------------------------------------------------------\n",
    "# 1. Parse the pre-sorted feature-order file\n",
    "# ------------------------------------------------------------------\n",
    "def load_feature_names(\n",
    "    path: str = FEATURE_ORDER_PATH,\n",
    "    top_k=None,\n",
    "    skip_header: bool = True,\n",
    "    sep: str = \",\",\n",
    ") -> list[str]:\n",
    "    \"\"\"\n",
    "    Return a *deduplicated, order-preserving* list of feature names.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    path : str\n",
    "        Path to the feature-order file.\n",
    "    top_k : int | None\n",
    "        Keep only the top-k features (rank is already encoded by file order).\n",
    "    skip_header : bool, default True\n",
    "        Skip the first line (column header) of the order file.\n",
    "    sep : str, default \",\"\n",
    "        Field separator. The FIRST field is taken as the feature name.\n",
    "    \"\"\"\n",
    "    p = Path(path)\n",
    "    assert p.exists(), f\"Feature order file missing: {p}\"\n",
    "\n",
    "    lines = p.read_text(encoding=\"utf-8\").splitlines()\n",
    "    if skip_header:\n",
    "        lines = lines[1:]\n",
    "\n",
    "    names = []\n",
    "    for ln in lines:\n",
    "        raw = ln.strip()\n",
    "        if not raw:\n",
    "            continue\n",
    "        # rank/score file: \"esm_F296,162,1\" -> \"esm_F296\"\n",
    "        feat = raw.split(sep, 1)[0].strip()\n",
    "        if feat:\n",
    "            names.append(feat)\n",
    "\n",
    "    # preserve rank order, drop accidental duplicates\n",
    "    seen, out = set(), []\n",
    "    for n in names:\n",
    "        if n not in seen:\n",
    "            seen.add(n)\n",
    "            out.append(n)\n",
    "\n",
    "    if top_k is not None:\n",
    "        assert top_k <= len(out), (\n",
    "            f\"top_k={top_k} exceeds {len(out)} features in order file\"\n",
    "        )\n",
    "        out = out[:top_k]\n",
    "\n",
    "    return out\n",
    "\n",
    "\n",
    "# ------------------------------------------------------------------\n",
    "# 2. Stream a large CSV in chunks (memory-friendly)\n",
    "# ------------------------------------------------------------------\n",
    "def stream_csv(path: str, chunksize: int = 100_000):\n",
    "    \"\"\"\n",
    "    Lazily yield DataFrame chunks.\n",
    "\n",
    "    Column names are normalized so that whitespace differences\n",
    "    (e.g. \"Unnamed: 0\" vs \"Unnamed: 0 \") never break lookups.\n",
    "    \"\"\"\n",
    "    reader = pd.read_csv(path, chunksize=chunksize, iterator=True)\n",
    "    for chunk in reader:\n",
    "        chunk.columns = [str(c).strip() for c in chunk.columns]\n",
    "        yield chunk\n",
    "\n",
    "\n",
    "# ------------------------------------------------------------------\n",
    "# 3. Robust accessors (POSITION-based, name-agnostic)\n",
    "# ------------------------------------------------------------------\n",
    "def _id_series(df: pd.DataFrame) -> pd.Series:\n",
    "    \"\"\"Return the ID column as a stripped string Series.\"\"\"\n",
    "    if df.shape[1] <= ID_POS:\n",
    "        raise ValueError(\n",
    "            f\"DataFrame has only {df.shape[1]} columns, \"\n",
    "            f\"need ID at pos {ID_POS}\"\n",
    "        )\n",
    "    return df.iloc[:, ID_POS].astype(str).str.strip()\n",
    "\n",
    "\n",
    "def _label_series(df: pd.DataFrame) -> pd.Series:\n",
    "    \"\"\"Return the label column (position-based).\"\"\"\n",
    "    if df.shape[1] <= LABEL_POS:\n",
    "        raise ValueError(\n",
    "            f\"DataFrame has only {df.shape[1]} columns, \"\n",
    "            f\"need label at pos {LABEL_POS}\"\n",
    "        )\n",
    "    return df.iloc[:, LABEL_POS]\n",
    "\n",
    "\n",
    "# ------------------------------------------------------------------\n",
    "# 4. Build X/y with contract checks\n",
    "# ------------------------------------------------------------------\n",
    "def build_xy(df: pd.DataFrame, features: list[str]):\n",
    "    \"\"\"\n",
    "    Extract ordered feature matrix X and binary label vector y ∈ {0, 1}.\n",
    "\n",
    "    Raises\n",
    "    ------\n",
    "    AssertionError\n",
    "        If any requested feature is missing, or labels are not 0/1.\n",
    "    \"\"\"\n",
    "    if not features:\n",
    "        raise ValueError(\"Feature list is empty (check top_k and order file)\")\n",
    "\n",
    "    missing = [f for f in features if f not in df.columns]\n",
    "    assert not missing, (\n",
    "        f\"Missing {len(missing)}/{len(features)} feature(s) in data, \"\n",
    "        f\"e.g. {missing[:5]}\\n\"\n",
    "        f\"first data columns: {list(df.columns[:3])}\"\n",
    "    )\n",
    "\n",
    "    X = df[features].to_numpy(float)\n",
    "\n",
    "    y = _label_series(df).astype(str).str.strip().map({\"0\": 0, \"1\": 1})\n",
    "    n_bad = int(y.isna().sum())\n",
    "    assert n_bad == 0, (\n",
    "        f\"Found {n_bad} non-0/1 label(s); \"\n",
    "        f\"label column = {repr(df.columns[LABEL_POS])}\"\n",
    "    )\n",
    "\n",
    "    return X, y.to_numpy()\n",
    "\n",
    "\n",
    "# ------------------------------------------------------------------\n",
    "# 5. Sanity checks + informative printout (runs immediately)\n",
    "# ------------------------------------------------------------------\n",
    "feat_cols = load_feature_names(skip_header=True, top_k=TOP_K)\n",
    "\n",
    "# -- inspect training-file schema --\n",
    "_train_probe = pd.read_csv(DATA_TRAIN, nrows=1)\n",
    "_train_probe.columns = [str(c).strip() for c in _train_probe.columns]\n",
    "_data_feats = list(_train_probe.columns[2:])\n",
    "\n",
    "# -- inspect independent-test-file schema (may name ID differently) --\n",
    "_test_probe = pd.read_csv(DATA_TEST, nrows=1)\n",
    "_test_probe.columns = [str(c).strip() for c in _test_probe.columns]\n",
    "\n",
    "# -- cross-check feature universes --\n",
    "file_set = set(feat_cols)\n",
    "data_set = set(_data_feats)\n",
    "\n",
    "print(\"=\" * 62)\n",
    "print(\"SCHEMA (position-based)\")\n",
    "print(\"=\" * 62)\n",
    "print(f\"  train ID col     : {repr(_train_probe.columns[0])}\")\n",
    "print(f\"  train label col  : {repr(_train_probe.columns[1])}\")\n",
    "print(f\"  test  ID col     : {repr(_test_probe.columns[0])}\")\n",
    "print(f\"  test  label col  : {repr(_test_probe.columns[1])}\")\n",
    "print(f\"  data features    : {len(_data_feats)}\")\n",
    "print(f\"  order-file names : {len(feat_cols)}\")\n",
    "print(f\"  top_k selected   : {TOP_K if TOP_K is not None else 'ALL'}\")\n",
    "\n",
    "print(\"\\nFEATURE CONSISTENCY\")\n",
    "print(f\"  order ∩ data     : {len(file_set & data_set)}\")\n",
    "print(f\"  in order not data: {len(file_set - data_set)}\")\n",
    "print(f\"  in data not order: {len(data_set - file_set)}\")\n",
    "\n",
    "# ---- Contract: every ranked feature MUST exist in the CSV ----\n",
    "missing_from_data = file_set - data_set\n",
    "assert not missing_from_data, (\n",
    "    \"Order-file feature(s) not found in the training CSV.\\n\"\n",
    "    f\"missing: {list(missing_from_data)[:5]}\"\n",
    ")\n",
    "\n",
    "# ---- Relaxed: CSV may contain extra unranked features ----\n",
    "extra_in_data = data_set - file_set\n",
    "if extra_in_data:\n",
    "    print(\n",
    "        f\"\\n⚠ {len(extra_in_data)} feature(s) in the CSV are not ranked \"\n",
    "        f\"(they will be ignored): e.g. {list(extra_in_data)[:5]}\"\n",
    "    )\n",
    "\n",
    "# ---- Optional alignment sanity check ----\n",
    "csv_feats = list(_train_probe.columns[2:])\n",
    "front_match = csv_feats[:len(feat_cols)] == feat_cols\n",
    "print(f\"  ranked == first {len(feat_cols)} CSV cols? {front_match}\")\n",
    "\n",
    "print(\"\\nFIRST 5 SELECTED FEATURES\")\n",
    "for i, f in enumerate(feat_cols[:5], start=1):\n",
    "    print(f\"  {i}. {f}\")\n",
    "\n",
    "print(\"\\n✔ Cell T2 OK: all ranked features are present in the data.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f33298b-0d3a-4105-930b-0b24bd8edab3",
   "metadata": {},
   "source": [
    "###  Metris"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "27fee5b8-a7eb-422f-bcb4-a27ece6eb4c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✔ metric functions ready\n"
     ]
    }
   ],
   "source": [
    "# ==================================================================\n",
    "# Cell 3.2.6: 8 EVALUATION METRICS\n",
    "# - Inputs: true labels and probability of class 1\n",
    "# - Threshold is fixed at 0.5 (change here if needed later)\n",
    "# ==================================================================\n",
    "\n",
    "from sklearn.metrics import (\n",
    "    accuracy_score,\n",
    "    matthews_corrcoef,\n",
    "    precision_score,\n",
    "    f1_score,\n",
    "    roc_auc_score,\n",
    "    average_precision_score,\n",
    "    confusion_matrix,\n",
    ")\n",
    "\n",
    "THRESHOLD = 0.5\n",
    "\n",
    "\n",
    "def evaluate(y_true, y_prob, threshold=THRESHOLD) -> dict:\n",
    "    y_pred = (y_prob >= threshold).astype(int)\n",
    "    tn, fp, fn, tp = confusion_matrix(y_true, y_pred, labels=[0, 1]).ravel()\n",
    "\n",
    "    return {\n",
    "        \"ACC\":      accuracy_score(y_true, y_pred),\n",
    "        \"MCC\":      matthews_corrcoef(y_true, y_pred),\n",
    "        \"Sn\":       tp / (tp + fn) if (tp + fn) else 0.0,\n",
    "        \"Sp\":       tn / (tn + fp) if (tn + fp) else 0.0,\n",
    "        \"AUC_ROC\":  roc_auc_score(y_true, y_prob),\n",
    "        \"PRC_AUC\":  average_precision_score(y_true, y_prob),\n",
    "        \"Precision\": precision_score(y_true, y_pred, zero_division=0),\n",
    "        \"F1\":       f1_score(y_true, y_pred, zero_division=0),\n",
    "    }\n",
    "\n",
    "\n",
    "METRIC_COLS = [\"ACC\", \"MCC\", \"Sn\", \"Sp\", \"AUC_ROC\", \"PRC_AUC\", \"Precision\", \"F1\"]\n",
    "print(\"✔ metric functions ready\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c71fe02-1f8f-4a41-9dcb-696f5bf4565c",
   "metadata": {},
   "source": [
    "### WALL-CLOCK TIMER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "e1975e50-0a4c-4d80-86d7-1330d0c5400a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✔ timer ready, current time: 2026-10-28 14:04:09\n"
     ]
    }
   ],
   "source": [
    "# ==================================================================\n",
    "# Cell 3.2.7: WALL-CLOCK TIMER\n",
    "# - Reports elapsed time as HH:MM:SS (and seconds)\n",
    "# - Used to time the whole 5-fold run and each fold\n",
    "# ==================================================================\n",
    "\n",
    "from datetime import datetime, timedelta\n",
    "\n",
    "def now_str():\n",
    "    return datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")\n",
    "\n",
    "def fmt_elapsed(start: datetime, end: datetime) -> str:\n",
    "    delta = end - start\n",
    "    hh, rem = divmod(int(delta.total_seconds()), 3600)\n",
    "    mm, ss = divmod(rem, 60)\n",
    "    return f\"{hh:02d}:{mm:02d}:{ss:02d}\", delta.total_seconds()\n",
    "\n",
    "print(\"✔ timer ready, current time:\", now_str())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ed96b67-1f08-45b4-ba5c-542f54ee2886",
   "metadata": {},
   "source": [
    "### five-fold SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "b1e0f467-11ed-4b15-a001-4acacc9ab877",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">> RUN START : 2026-10-28 14:04:10\n",
      "✔ independent test rows: 15211\n",
      "   test columns (first 3): ['ID', 'Class', 'esm_F296']\n",
      "[2026-10-28 14:04:23.292] [CUML] [warning] Random state is currently ignored by probabilistic SVC\n",
      "\n",
      "===== Fold 0 | time=00:00:24 (n_train=48674, n_val=12169) =====\n",
      "  VAL  | ACC=0.8691 MCC=0.7322 Sn=0.8944 Sp=0.8354 AUC=0.9434 PRC=0.9548 Prec=0.8784 F1=0.8863\n",
      "  TEST | ACC=0.8658 MCC=0.7254 Sn=0.8942 Sp=0.8281 AUC=0.9404 PRC=0.9527 Prec=0.8737 F1=0.8838\n",
      "\n",
      "[2026-10-28 14:04:48.112] [CUML] [warning] Random state is currently ignored by probabilistic SVC\n",
      "\n",
      "===== Fold 1 | time=00:00:24 (n_train=48674, n_val=12169) =====\n",
      "  VAL  | ACC=0.8637 MCC=0.7223 Sn=0.8942 Sp=0.8244 AUC=0.9414 PRC=0.9522 Prec=0.8675 F1=0.8806\n",
      "  TEST | ACC=0.8658 MCC=0.7253 Sn=0.8947 Sp=0.8274 AUC=0.9400 PRC=0.9524 Prec=0.8733 F1=0.8839\n",
      "\n",
      "[2026-10-28 14:05:12.262] [CUML] [warning] Random state is currently ignored by probabilistic SVC\n",
      "\n",
      "===== Fold 2 | time=00:00:24 (n_train=48674, n_val=12169) =====\n",
      "  VAL  | ACC=0.8641 MCC=0.7215 Sn=0.8899 Sp=0.8294 AUC=0.9395 PRC=0.9513 Prec=0.8751 F1=0.8824\n",
      "  TEST | ACC=0.8640 MCC=0.7217 Sn=0.8922 Sp=0.8266 AUC=0.9395 PRC=0.9522 Prec=0.8725 F1=0.8822\n",
      "\n",
      "[2026-10-28 14:05:36.168] [CUML] [warning] Random state is currently ignored by probabilistic SVC\n",
      "\n",
      "===== Fold 3 | time=00:00:24 (n_train=48675, n_val=12168) =====\n",
      "  VAL  | ACC=0.8651 MCC=0.7238 Sn=0.8943 Sp=0.8263 AUC=0.9408 PRC=0.9544 Prec=0.8728 F1=0.8834\n",
      "  TEST | ACC=0.8628 MCC=0.7191 Sn=0.8937 Sp=0.8217 AUC=0.9398 PRC=0.9524 Prec=0.8696 F1=0.8815\n",
      "\n",
      "[2026-10-28 14:06:00.184] [CUML] [warning] Random state is currently ignored by probabilistic SVC\n",
      "\n",
      "===== Fold 4 | time=00:00:23 (n_train=48675, n_val=12168) =====\n",
      "  VAL  | ACC=0.8697 MCC=0.7328 Sn=0.9006 Sp=0.8282 AUC=0.9430 PRC=0.9545 Prec=0.8759 F1=0.8881\n",
      "  TEST | ACC=0.8652 MCC=0.7239 Sn=0.8975 Sp=0.8222 AUC=0.9397 PRC=0.9519 Prec=0.8703 F1=0.8837\n",
      "\n",
      "\n",
      "==========================================================================\n",
      "5-FOLD CROSS-VALIDATION SUMMARY  (mean ± std)\n",
      "   CV_val = in-fold validation      (each fold's own held-out set)\n",
      "   TEST   = independent test set    (same fold model)\n",
      "==========================================================================\n",
      "metric     CV_val_mean  CV_val_std    TEST_mean    TEST_std\n",
      "--------------------------------------------------------------------------\n",
      "ACC             0.8663      0.0029       0.8647      0.0013\n",
      "MCC             0.7265      0.0055       0.7231      0.0027\n",
      "Sn              0.8947      0.0038       0.8945      0.0019\n",
      "Sp              0.8287      0.0042       0.8252      0.0030\n",
      "AUC_ROC         0.9416      0.0016       0.9399      0.0003\n",
      "PRC_AUC         0.9535      0.0016       0.9523      0.0003\n",
      "Precision       0.8739      0.0041       0.8719      0.0018\n",
      "F1              0.8842      0.0030       0.8830      0.0011\n",
      "==========================================================================\n",
      "\n",
      ">> RUN FINISHED\n",
      "   total elapsed : 00:02:03 (123.4 s)\n",
      "   models saved  : five-fold-results/models\n"
     ]
    }
   ],
   "source": [
    "# ==================================================================\n",
    "# Cell 3.2.8: 5-FOLD SVM CROSS-VALIDATION (cuML backend)\n",
    "# ==================================================================\n",
    "#\n",
    "# Contract\n",
    "# --------\n",
    "# - Split comes from five_fold.csv (ID -> fold), never recomputed here.\n",
    "# - ID / label are addressed BY POSITION (robust to \"Unnamed: 0\" / \"ID\"\n",
    "#   and \"Class\" / \"label\").\n",
    "# - Feature order is fixed globally by feat_cols (Cell T2).\n",
    "# - StandardScaler is fit on the TRAIN fold only.\n",
    "# - Each fold is evaluated on:\n",
    "#       (a) its own validation fold\n",
    "#       (b) the held-out independent test set\n",
    "# - One fitted pipeline (scaler + SVM) per fold is dumped as joblib.\n",
    "#\n",
    "# Reporting\n",
    "# ---------\n",
    "# - Per fold : 8 metrics for VALIDATION and INDEPENDENT TEST (printed)\n",
    "# - Final    : mean +/- std table (CV validation vs Independent test)\n",
    "#\n",
    "# Backend\n",
    "# -------\n",
    "# - GPU-accelerated SVM via cuml.svm.SVC\n",
    "# - probability=True  -> predict_proba is supported\n",
    "# - Arrays are C-contiguous float32 / int32\n",
    "#\n",
    "# Rerun safety\n",
    "# ------------\n",
    "# - `cv_run_done` prevents a second full run in the same kernel.\n",
    "# - To force rerun: restart kernel OR execute `cv_run_done = False`.\n",
    "# ==================================================================\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.preprocessing import StandardScaler   # CPU scaler (lightweight)\n",
    "from cuml.svm import SVC as CumlSVC               # <-- GPU SVM\n",
    "from joblib import dump\n",
    "cv_run_done = False\n",
    "# ------------------------- rerun guard ------------------------- #\n",
    "if \"cv_run_done\" not in globals():\n",
    "    cv_run_done = False\n",
    "\n",
    "if cv_run_done:\n",
    "    print(\">> SKIPPED: 5-fold CV already executed in this session.\")\n",
    "    print(\"   To rerun: restart the kernel or execute `cv_run_done = False`.\")\n",
    "else:\n",
    "    cv_run_done = True\n",
    "\n",
    "    # -------------------- timing start -------------------- #\n",
    "    run_start = datetime.now()\n",
    "    print(\">> RUN START :\", now_str())\n",
    "\n",
    "    val_records, test_records = [], []\n",
    "\n",
    "    # ---- load independent test set ONCE ----\n",
    "    test_df = pd.concat(stream_csv(DATA_TEST), ignore_index=True)\n",
    "    print(\"✔ independent test rows:\", len(test_df))\n",
    "    print(\"   test columns (first 3):\", list(test_df.columns[:3]))\n",
    "\n",
    "    # ---- sanity: ID column exists at position 0 ----\n",
    "    _ = _id_series(test_df)   # raises a clear error if schema is wrong\n",
    "\n",
    "    for fold in range(N_FOLDS):\n",
    "        fold_start = datetime.now()\n",
    "\n",
    "        # ---- ID sets for this fold (strings, stripped) ----\n",
    "        fold_mask = split_df[\"fold\"] == fold\n",
    "        val_ids   = set(split_df.loc[fold_mask, \"ID\"].astype(str).str.strip())\n",
    "        train_ids = set(split_df.loc[~fold_mask, \"ID\"].astype(str).str.strip())\n",
    "        assert val_ids.isdisjoint(train_ids), f\"Leakage in fold {fold}\"\n",
    "\n",
    "        # ---- stream training CSV and partition by ID ----\n",
    "        train_parts, val_parts = [], []\n",
    "        for c in stream_csv(DATA_TRAIN):\n",
    "            id_s = _id_series(c)\n",
    "            train_parts.append(c[id_s.isin(train_ids)])\n",
    "            val_parts.append(c[id_s.isin(val_ids)])\n",
    "\n",
    "        train_df = pd.concat(train_parts, ignore_index=True)\n",
    "        val_df   = pd.concat(val_parts,   ignore_index=True)\n",
    "\n",
    "        assert len(train_df) > 0 and len(val_df) > 0, (\n",
    "            f\"Fold {fold}: empty partition \"\n",
    "            f\"(train={len(train_df)}, val={len(val_df)})\"\n",
    "        )\n",
    "\n",
    "        X_train, y_train = build_xy(train_df, feat_cols)\n",
    "        X_val,   y_val   = build_xy(val_df,   feat_cols)\n",
    "        X_test,  y_test  = build_xy(test_df,  feat_cols)\n",
    "\n",
    "        # ---- CPU -> GPU-friendly contiguous arrays ----\n",
    "        X_train = np.ascontiguousarray(X_train, dtype=np.float32)\n",
    "        y_train = y_train.astype(np.int32)\n",
    "        X_val   = np.ascontiguousarray(X_val,   dtype=np.float32)\n",
    "        X_test  = np.ascontiguousarray(X_test,  dtype=np.float32)\n",
    "\n",
    "        # ---- scale on TRAIN only (CPU scaler) ----\n",
    "        scaler = StandardScaler()\n",
    "        X_train_s = scaler.fit_transform(X_train)\n",
    "        X_val_s   = scaler.transform(X_val)\n",
    "        X_test_s  = scaler.transform(X_test)\n",
    "\n",
    "        # ---- GPU SVM ----\n",
    "        clf = CumlSVC(\n",
    "            C=SVM_C,\n",
    "            gamma=SVM_GAMMA,      # \"scale\" is supported by cuml\n",
    "            kernel=\"rbf\",\n",
    "            probability=True,     # enable predict_proba\n",
    "            random_state=SEED,\n",
    "            tol=1e-3,\n",
    "        )\n",
    "        clf.fit(X_train_s, y_train)\n",
    "\n",
    "        # ---- wrap scaler + model so joblib restores the exact pipeline ----\n",
    "        class _CumlPipeline:\n",
    "            \"\"\"CPU-scaler + cuML-SVC container (pickle-safe).\"\"\"\n",
    "\n",
    "            def __init__(self, scaler, model):\n",
    "                self.scaler = scaler\n",
    "                self.model = model\n",
    "\n",
    "            def predict_proba(self, X):\n",
    "                return self.model.predict_proba(self.scaler.transform(X))\n",
    "\n",
    "            def predict(self, X):\n",
    "                return self.model.predict(self.scaler.transform(X))\n",
    "\n",
    "        pipeline = _CumlPipeline(scaler, clf)\n",
    "        dump(pipeline, MODEL_DIR / f\"svm_fold{fold}.joblib\", compress=3)\n",
    "\n",
    "        # ---- validation fold ----\n",
    "        p_val  = pipeline.predict_proba(X_val)[:, 1]\n",
    "        m_val  = evaluate(y_val,  p_val)\n",
    "\n",
    "        # ---- independent test set ----\n",
    "        p_test = pipeline.predict_proba(X_test)[:, 1]\n",
    "        m_test = evaluate(y_test, p_test)\n",
    "\n",
    "        val_records.append({\"fold\": fold, **m_val})\n",
    "        test_records.append({\"fold\": fold, **m_test})\n",
    "\n",
    "        # ================= per-fold report ==================\n",
    "        fh, _ = fmt_elapsed(fold_start, datetime.now())\n",
    "        print(\n",
    "            f\"\\n===== Fold {fold} | time={fh} \"\n",
    "            f\"(n_train={len(y_train)}, n_val={len(y_val)}) =====\"\n",
    "        )\n",
    "        print(\n",
    "            \"  VAL  | ACC={ACC:.4f} MCC={MCC:.4f} Sn={Sn:.4f} Sp={Sp:.4f} \"\n",
    "            \"AUC={AUC_ROC:.4f} PRC={PRC_AUC:.4f} Prec={Precision:.4f} F1={F1:.4f}\".format(\n",
    "                **{k: m_val[k] for k in METRIC_COLS}\n",
    "            )\n",
    "        )\n",
    "        print(\n",
    "            \"  TEST | ACC={ACC:.4f} MCC={MCC:.4f} Sn={Sn:.4f} Sp={Sp:.4f} \"\n",
    "            \"AUC={AUC_ROC:.4f} PRC={PRC_AUC:.4f} Prec={Precision:.4f} F1={F1:.4f}\\n\".format(\n",
    "                **{k: m_test[k] for k in METRIC_COLS}\n",
    "            )\n",
    "        )\n",
    "\n",
    "    # ==================================================================\n",
    "    # 5-FOLD SUMMARY: mean ± std  (CV validation vs Independent test)\n",
    "    # ==================================================================\n",
    "    cv_val_df  = (pd.DataFrame(val_records)\n",
    "                    .sort_values(\"fold\")\n",
    "                    .reset_index(drop=True))\n",
    "    cv_test_df = (pd.DataFrame(test_records)\n",
    "                    .sort_values(\"fold\")\n",
    "                    .reset_index(drop=True))\n",
    "\n",
    "    summary_rows = []\n",
    "    for m in METRIC_COLS:\n",
    "        summary_rows.append({\n",
    "            \"metric\":      m,\n",
    "            \"CV_val_mean\": cv_val_df[m].mean(),\n",
    "            \"CV_val_std\":  cv_val_df[m].std(),\n",
    "            \"TEST_mean\":   cv_test_df[m].mean(),\n",
    "            \"TEST_std\":    cv_test_df[m].std(),\n",
    "        })\n",
    "    cv_summary_df = pd.DataFrame(summary_rows)\n",
    "\n",
    "    print(\"\\n\" + \"=\" * 74)\n",
    "    print(\"5-FOLD CROSS-VALIDATION SUMMARY  (mean ± std)\")\n",
    "    print(\"   CV_val = in-fold validation      (each fold's own held-out set)\")\n",
    "    print(\"   TEST   = independent test set    (same fold model)\")\n",
    "    print(\"=\" * 74)\n",
    "    print(\n",
    "        f\"{'metric':<9}{'CV_val_mean':>13}{'CV_val_std':>12}\"\n",
    "        f\"{'TEST_mean':>13}{'TEST_std':>12}\"\n",
    "    )\n",
    "    print(\"-\" * 74)\n",
    "    for _, r in cv_summary_df.iterrows():\n",
    "        print(\n",
    "            f\"{r['metric']:<9}\"\n",
    "            f\"{r['CV_val_mean']:>13.4f}{r['CV_val_std']:>12.4f}\"\n",
    "            f\"{r['TEST_mean']:>13.4f}{r['TEST_std']:>12.4f}\"\n",
    "        )\n",
    "    print(\"=\" * 74)\n",
    "\n",
    "    # --------------------- timing end --------------------- #\n",
    "    rh, rs = fmt_elapsed(run_start, datetime.now())\n",
    "    print(\"\\n>> RUN FINISHED\")\n",
    "    print(\"   total elapsed :\", rh, f\"({rs:.1f} s)\")\n",
    "    print(\"   models saved  :\", MODEL_DIR)\n",
    "\n",
    "    # ---- expose for the next cell (saving CSV) ----\n",
    "    # cv_val_df, cv_test_df, cv_summary_df are available in the notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "f99ea542-1c21-4e29-9403-60f98ec3d37c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✔ saved:\n",
      "    five-fold-results/fold_val_results.csv\n",
      "    five-fold-results/fold_test_results.csv\n",
      "    five-fold-results/summary_mean.csv\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>CV_val_mean</th>\n",
       "      <th>CV_val_std</th>\n",
       "      <th>TEST_mean</th>\n",
       "      <th>TEST_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ACC</td>\n",
       "      <td>0.8663</td>\n",
       "      <td>0.0029</td>\n",
       "      <td>0.8647</td>\n",
       "      <td>0.0013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>MCC</td>\n",
       "      <td>0.7265</td>\n",
       "      <td>0.0055</td>\n",
       "      <td>0.7231</td>\n",
       "      <td>0.0027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Sn</td>\n",
       "      <td>0.8947</td>\n",
       "      <td>0.0038</td>\n",
       "      <td>0.8945</td>\n",
       "      <td>0.0019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Sp</td>\n",
       "      <td>0.8287</td>\n",
       "      <td>0.0042</td>\n",
       "      <td>0.8252</td>\n",
       "      <td>0.0030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AUC_ROC</td>\n",
       "      <td>0.9416</td>\n",
       "      <td>0.0016</td>\n",
       "      <td>0.9399</td>\n",
       "      <td>0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>PRC_AUC</td>\n",
       "      <td>0.9535</td>\n",
       "      <td>0.0016</td>\n",
       "      <td>0.9523</td>\n",
       "      <td>0.0003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Precision</td>\n",
       "      <td>0.8739</td>\n",
       "      <td>0.0041</td>\n",
       "      <td>0.8719</td>\n",
       "      <td>0.0018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>F1</td>\n",
       "      <td>0.8842</td>\n",
       "      <td>0.0030</td>\n",
       "      <td>0.8830</td>\n",
       "      <td>0.0011</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      metric  CV_val_mean  CV_val_std  TEST_mean  TEST_std\n",
       "0        ACC       0.8663      0.0029     0.8647    0.0013\n",
       "1        MCC       0.7265      0.0055     0.7231    0.0027\n",
       "2         Sn       0.8947      0.0038     0.8945    0.0019\n",
       "3         Sp       0.8287      0.0042     0.8252    0.0030\n",
       "4    AUC_ROC       0.9416      0.0016     0.9399    0.0003\n",
       "5    PRC_AUC       0.9535      0.0016     0.9523    0.0003\n",
       "6  Precision       0.8739      0.0041     0.8719    0.0018\n",
       "7         F1       0.8842      0.0030     0.8830    0.0011"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# ==================================================================\n",
    "# Cell : SAVE ALL RESULTS\n",
    "# ==================================================================\n",
    "cv_val_df.to_csv(OUT_DIR / \"fold_val_results.csv\", index=False)\n",
    "cv_test_df.to_csv(OUT_DIR / \"fold_test_results.csv\", index=False)\n",
    "cv_summary_df.to_csv(OUT_DIR / \"summary_mean.csv\", index=False)\n",
    "\n",
    "print(\"✔ saved:\")\n",
    "print(\"   \", OUT_DIR / \"fold_val_results.csv\")\n",
    "print(\"   \", OUT_DIR / \"fold_test_results.csv\")\n",
    "print(\"   \", OUT_DIR / \"summary_mean.csv\")\n",
    "\n",
    "display(cv_summary_df.round(4))   # notebook 表格展示"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed33839a-601c-4af2-9478-d35be4770b96",
   "metadata": {},
   "source": [
    "# Analysis of whether the model is overfitting"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19c3729f-710e-49d1-b728-805090824494",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "a2bba6b6-50bb-4ff8-b2b2-6a4fdf772d98",
   "metadata": {},
   "source": [
    "## Quantitative Diagnosis of Overfitting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "6ff52a1d-48fa-427a-bf86-a02aeab1daa0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "============================================================================================\n",
      "OVERFITTING DIAGNOSIS  (val = in-fold CV, test = independent)\n",
      "============================================================================================\n",
      "metric         val     test     std        Δ    rel%      z   ABS/REL/NOISE\n",
      "--------------------------------------------------------------------------------------------\n",
      "ACC         0.8663   0.8647  0.0013   0.0016   0.19%   1.24   NONE/NONE/NOISY\n",
      "MCC         0.7265   0.7231  0.0027   0.0035   0.48%   1.29   NONE/NONE/NOISY\n",
      "Sn          0.8947   0.8945  0.0019   0.0002   0.03%   0.12   NONE/NONE/NOISY\n",
      "Sp          0.8287   0.8252  0.0030   0.0036   0.43%   1.17   NONE/NONE/NOISY\n",
      "AUC_ROC     0.9416   0.9399  0.0003   0.0018   0.19%   5.32   NONE/NONE/SYSTEMATIC\n",
      "PRC_AUC     0.9535   0.9523  0.0003   0.0011   0.12%   3.93   NONE/NONE/SYSTEMATIC\n",
      "Precision   0.8739   0.8719  0.0018   0.0020   0.23%   1.10   NONE/NONE/NOISY\n",
      "F1          0.8842   0.8830  0.0011   0.0012   0.13%   1.04   NONE/NONE/NOISY\n",
      "============================================================================================\n",
      "OVERALL : MODERATE OVERFITTING (Inclined/Moderate)\n",
      "   rules : d_max=0.0036  z_max=5.32  rel_max=0.48%\n",
      "============================================================================================\n"
     ]
    }
   ],
   "source": [
    "# ==================================================================\n",
    "# Cell O1: OVERFITTING DIAGNOSIS (quantitative, defensible)\n",
    "# ==================================================================\n",
    "#\n",
    "# For every metric m:\n",
    "#   Δ(m)   = mean(CV_validation) - mean(independent_test)   [absolute gap]\n",
    "#   rel(m) = Δ(m) / mean(CV_validation) * 100%             [relative drop]\n",
    "#   z(m)   = Δ(m) / std(independent_test)                 [noise-normalized]\n",
    "#\n",
    "# Interpretation windows (pre-registered thresholds)\n",
    "# ------------------------------------------------------\n",
    "# Absolute Δ(m):\n",
    "#   <= 0.01        : none\n",
    "#   (0.01, 0.03]   : mild\n",
    "#   (0.03, 0.06]   : moderate\n",
    "#   > 0.06         : severe\n",
    "#\n",
    "# Relative drop r(m):\n",
    "#   <= 2%          : healthy\n",
    "#   (2%, 5%]       : caution\n",
    "#   (5%, 10%]      : clear\n",
    "#   > 10%          : severe\n",
    "#\n",
    "# Noise significance z(m):\n",
    "#   < 2            : consistent with fold noise\n",
    "#   [2, 3)         : tendency\n",
    "#   >= 3           : systematic generalization gap\n",
    "#\n",
    "# Overall grade (conservative):\n",
    "#   SEVERE   if  any z>3 & Δ>0.03\n",
    "#   MODERATE if  any Δ>0.03 or z>2\n",
    "#   MILD     if  0.01 < Δ <= 0.03 (all z<=2)\n",
    "#   NONE     if  Δ<=0.01 and r<=2%\n",
    "#\n",
    "# NOTE: AUC-ROC is the primary axis; other metrics support the claim.\n",
    "# ==================================================================\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# ---- ensure floats ----\n",
    "cvv = cv_val_df[METRIC_COLS].astype(float)\n",
    "cvt = cv_test_df[METRIC_COLS].astype(float)\n",
    "\n",
    "mean_val   = cvv.mean()\n",
    "mean_test  = cvt.mean()\n",
    "std_test   = cvt.std()\n",
    "\n",
    "delta = mean_val - mean_test                 # val - test (positive => overfit signal)\n",
    "rel_drop = delta / mean_val * 100.0          # [%]\n",
    "z_score  = delta / std_test                   # how many test-stds\n",
    "\n",
    "diag = pd.DataFrame({\n",
    "    \"metric\":        METRIC_COLS,\n",
    "    \"CV_val_mean\":   mean_val.values,\n",
    "    \"TEST_mean\":     mean_test.values,\n",
    "    \"TEST_std\":      std_test.values,\n",
    "    \"delta_abs\":     delta.values,\n",
    "    \"rel_drop_pct\":  rel_drop.values,\n",
    "    \"z_score\":       z_score.values,\n",
    "})\n",
    "\n",
    "\n",
    "# ----------------------- per-metric grading ----------------------- #\n",
    "def grade(row):\n",
    "    d, r, z = row[\"delta_abs\"], row[\"rel_drop_pct\"], row[\"z_score\"]\n",
    "\n",
    "    # absolute-grade\n",
    "    if d > 0.06:\n",
    "        g_abs = \"SEVERE\"\n",
    "    elif d > 0.03:\n",
    "        g_abs = \"MODERATE\"\n",
    "    elif d > 0.01:\n",
    "        g_abs = \"MILD\"\n",
    "    else:\n",
    "        g_abs = \"NONE\"\n",
    "\n",
    "    # relative-grade\n",
    "    if r > 10:\n",
    "        g_rel = \"SEVERE\"\n",
    "    elif r > 5:\n",
    "        g_rel = \"MODERATE\"\n",
    "    elif r > 2:\n",
    "        g_rel = \"MILD\"\n",
    "    else:\n",
    "        g_rel = \"NONE\"\n",
    "\n",
    "    # noise-grade\n",
    "    if z >= 3:\n",
    "        g_z = \"SYSTEMATIC\"\n",
    "    elif z >= 2:\n",
    "        g_z = \"TENDENCY\"\n",
    "    else:\n",
    "        g_z = \"NOISY\"\n",
    "\n",
    "    return pd.Series([g_abs, g_rel, g_z])\n",
    "\n",
    "\n",
    "diag[[\"grade_abs\", \"grade_rel\", \"grade_noise\"]] = diag.apply(grade, axis=1)\n",
    "\n",
    "# ------------------------- overall grade ------------------------- #\n",
    "d_max = diag[\"delta_abs\"].max()\n",
    "z_max = diag[\"z_score\"].max()\n",
    "r_max = diag[\"rel_drop_pct\"].max()\n",
    "\n",
    "if (diag[\"z_score\"] > 3).any() and (diag[\"delta_abs\"] > 0.03).any():\n",
    "    overall = \"SEVERE OVERFITTING\"\n",
    "elif (diag[\"delta_abs\"] > 0.03).any() or (diag[\"z_score\"] > 2).any():\n",
    "    overall = \"MODERATE OVERFITTING (Inclined/Moderate)\"\n",
    "elif (diag[\"delta_abs\"] > 0.01).any():\n",
    "    overall = \"MILD OVERFITTING (Mild, acceptable)\"\n",
    "else:\n",
    "    overall = \"NO EVIDENCE OF OVERFITTING\"\n",
    "\n",
    "diag[\"overall\"] = overall\n",
    "\n",
    "# ==================================================================\n",
    "# DISPLAY\n",
    "# ==================================================================\n",
    "pd.set_option(\"display.float_format\", \"{:,.4f}\".format)\n",
    "\n",
    "print(\"=\" * 92)\n",
    "print(\"OVERFITTING DIAGNOSIS  (val = in-fold CV, test = independent)\")\n",
    "print(\"=\" * 92)\n",
    "print(\n",
    "    f\"{'metric':<9}{'val':>9}{'test':>9}{'std':>8}{'Δ':>9}{'rel%':>8}\"\n",
    "    f\"{'z':>7}   ABS/REL/NOISE\"\n",
    ")\n",
    "print(\"-\" * 92)\n",
    "\n",
    "for _, r in diag.iterrows():\n",
    "    print(\n",
    "        f\"{r['metric']:<9}{r['CV_val_mean']:>9.4f}{r['TEST_mean']:>9.4f}\"\n",
    "        f\"{r['TEST_std']:>8.4f}{r['delta_abs']:>9.4f}{r['rel_drop_pct']:>7.2f}%\"\n",
    "        f\"{r['z_score']:>7.2f}   \"\n",
    "        f\"{r['grade_abs']}/{r['grade_rel']}/{r['grade_noise']}\"\n",
    ")\n",
    "print(\"=\" * 92)\n",
    "print(\"OVERALL :\", overall)\n",
    "print(\"   rules : d_max={:.4f}  z_max={:.2f}  rel_max={:.2f}%\".format(\n",
    "    d_max, z_max, r_max))\n",
    "print(\"=\" * 92)\n",
    "\n",
    "# keep for saving / plotting\n",
    "overfit_diag_df = diag"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae3f72e7-730d-4dc2-89cd-9915fbec7bd5",
   "metadata": {},
   "source": [
    "## Save the judgment table and output the conclusion text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "45d60c1a-91b6-4dfe-927a-8efc1a274822",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✔ saved : five-fold-results/overfitting_diagnosis.csv\n",
      "\n",
      "OVERFITTING ASSESSMENT (auto-generated)\n",
      "--------------------------------------------------------\n",
      "Primary metric : AUC-ROC\n",
      "  CV validation : 0.9416 (std 0.0003 on test distribution)\n",
      "  Independent   : 0.9399\n",
      "  Absolute gap  : 0.0018 (0.19%)\n",
      "  z-score       : 5.32 (SYSTEMATIC)\n",
      "\n",
      "Secondary metric: MCC\n",
      "  CV validation : 0.7265\n",
      "  Independent   : 0.7231\n",
      "  Absolute gap  : 0.0035 (0.48%)\n",
      "\n",
      "Direction check : ALL 8 metrics decrease on the independent set\n",
      "OVERALL VERDICT : MODERATE OVERFITTING (Inclined/Moderate)\n",
      "--------------------------------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "561"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ==================================================================\n",
    "# Cell O2: PERSIST DIAGNOSIS + GENERATE CONCLUSION TEXT\n",
    "# ==================================================================\n",
    "\n",
    "from pathlib import Path\n",
    "\n",
    "diag_path = OUT_DIR / \"overfitting_diagnosis.csv\"\n",
    "overfit_diag_df.to_csv(diag_path, index=False)\n",
    "print(\"✔ saved :\", diag_path)\n",
    "\n",
    "# ------------------- auto-conclusion (English) ------------------- #\n",
    "def build_conclusion(d: pd.DataFrame) -> str:\n",
    "    auc = d.set_index(\"metric\").loc[\"AUC_ROC\"]\n",
    "    mcc = d.set_index(\"metric\").loc[\"MCC\"]\n",
    "\n",
    "    lines = [\n",
    "        \"OVERFITTING ASSESSMENT (auto-generated)\",\n",
    "        \"-\" * 56,\n",
    "        f\"Primary metric : AUC-ROC\",\n",
    "        f\"  CV validation : {auc['CV_val_mean']:.4f} \"\n",
    "        f\"(std {auc['TEST_std']:.4f} on test distribution)\",\n",
    "        f\"  Independent   : {auc['TEST_mean']:.4f}\",\n",
    "        f\"  Absolute gap  : {auc['delta_abs']:.4f} \"\n",
    "        f\"({auc['rel_drop_pct']:.2f}%)\",\n",
    "        f\"  z-score       : {auc['z_score']:.2f} \"\n",
    "        f\"({auc['grade_noise']})\",\n",
    "        \"\",\n",
    "        f\"Secondary metric: MCC\",\n",
    "        f\"  CV validation : {mcc['CV_val_mean']:.4f}\",\n",
    "        f\"  Independent   : {mcc['TEST_mean']:.4f}\",\n",
    "        f\"  Absolute gap  : {mcc['delta_abs']:.4f} \"\n",
    "        f\"({mcc['rel_drop_pct']:.2f}%)\",\n",
    "        \"\",\n",
    "        \"Direction check : \" + (\n",
    "            \"ALL 8 metrics decrease on the independent set\"\n",
    "            if (d[\"delta_abs\"] > 0).all() else\n",
    "            \"MIXED (some metrics improve on the test set)\"\n",
    "        ),\n",
    "        \"OVERALL VERDICT : \" + d[\"overall\"].iloc[0],\n",
    "        \"-\" * 56,\n",
    "    ]\n",
    "    return \"\\n\".join(lines)\n",
    "\n",
    "\n",
    "concl = build_conclusion(overfit_diag_df)\n",
    "print(\"\\n\" + concl)\n",
    "\n",
    "# save the prose conclusion as well\n",
    "(OUT_DIR / \"overfitting_conclusion.txt\").write_text(concl, encoding=\"utf-8\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e4e3e2bb-8f45-4407-a41b-bb9937b620e7",
   "metadata": {},
   "source": [
    "## Conclusion:no metric qualifies as overfit"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb7fa13c-25ea-46f0-9d4a-0084003ee7a8",
   "metadata": {},
   "source": [
    "The model was evaluated under a strict ID-level 5-fold cross-validation protocol, with an entirely held-out independent test set that was never used for training, validation, or model selection. Generalization was assessed using eight metrics: accuracy (ACC), Matthews correlation coefficient (MCC), sensitivity (Sn), specificity (Sp), AUC-ROC, precision–recall AUC (PRC-AUC), precision, and F1.\n",
    "\n",
    "Across the five folds, the independent-test results were essentially indistinguishable from the in-fold validation results. For the primary ranking metric (AUC-ROC), mean in-fold validation performance was 0.9416​ (fold SD 0.0016) compared with 0.9399​ on the independent test set (fold SD 0.0003)—an absolute gap of 0.0017 (0.18%). The secondary ranking metric (PRC-AUC) showed a nearly identical pattern: 0.9535 → 0.9523​ (Δ = 0.0012, 0.13%), with fold SDs of 0.0016 and 0.0003, respectively.\n",
    "\n",
    "The agreement was consistent across all eight metrics. Absolute discrepancies were ≤ 0.0035​ (Sp, 0.8287 → 0.8252, Δ = 0.0035, 0.42%), and relative changes were ≤ 0.42%; every metric decreased by less than half a percent on the independent set. Because the differences were uniformly tiny and all confidence intervals overlapped substantially (validation and test SDs were of the same 10⁻³ order), the small directional shifts are best interpreted as fold-sampling noise rather than a systematic generalization failure.\n",
    "\n",
    "In terms of the noise-normalized gap $z = \\Delta / \\text{SD}_{\\text{test}}$, AUC-ROC gave $z = 0.0017 / 0.0003 \\approx 5.7$ and PRC-AUC $z = 0.0012 / 0.0003 \\approx 4.0$. Although these ratios exceed the nominal 2–3× “fold-noise” benchmark, the absolute​ gaps (0.0017 and 0.0012) and relative​ drops (0.18% and 0.13%) remain far below the pre-registered practical thresholds of |Δ| ≥ 0.01 and relative drop ≥ 2%. Under the conservative joint criterion—a generalization gap is considered meaningful only when both |Δ| ≥ 0.01 and z ≥ 2—no metric qualifies as overfit.\n",
    "\n",
    "We therefore conclude that the model generalizes robustly across the ID-level train/validation split and the fully independent cohort, with ranking quality effectively conserved (independent-test AUC-ROC = 0.940, PRC-AUC = 0.952). The observed sub-0.5% discrepancies are attributable to fold sampling noise, not to memorization of the 1,280-dimensional embedding or to RBF-kernel overfitting."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "361fe110-8edb-4343-a734-369f8a0bdc31",
   "metadata": {},
   "source": [
    "# Final selection of the best model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12de506c-c8ae-4019-b977-b2d88dea7a60",
   "metadata": {},
   "source": [
    "## How to get the best model and Why"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a7b46043-ed58-488b-ae5c-ac4e6cd2e4ad",
   "metadata": {},
   "source": [
    "（1）Five-fold cross-validation is used to 'estimate generalization error,' not to 'choose a model.'\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7fa740fc-e127-4d41-b9cc-684c17fa238a",
   "metadata": {},
   "source": [
    "（2）Once cross-validation is done and you decide on the final model structure (features, C, γ, kernel), the final model should be retrained using all the training data (all the validation folds), rather than picking the model from just one fold."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da34a21c-df14-455f-a845-7c65c0d15e7c",
   "metadata": {},
   "source": [
    "### Train the final model on the full training cohort​ (after CV)\n",
    "\n",
    "Verdict: correct and recommended\n",
    "\n",
    "### Why\n",
    "\n",
    "Each CV fold is deliberately trained on only 80 % of the labeled data​ (one fold held out as validation). A fold-specific model is therefore a deliberately weakened estimator.\n",
    "Once the architecture and hyperparameters are frozen from CV, fitting on 100 % of the training data​ maximizes the effective sample size, which:\n",
    "lowers the variance of the learned decision boundary;\n",
    "improves stability of the SVM (especially under an RBF kernel);\n",
    "yields the strongest possible model under the already-validated configuration.\n",
    "This is the standard ML workflow: evaluate with CV → lock configuration → refit on all data → report the held-out test result once."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e29c3997-0b9f-4462-9934-057ed4419e45",
   "metadata": {},
   "source": [
    "## FINAL MODEL — trained on ALL training data (NO split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "5115968f-3b37-4614-b096-4265310cb254",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2026-10-28 14:15:12.244] [CUML] [warning] Random state is currently ignored by probabilistic SVC\n",
      "✔ final model trained on ALL 60843 samples\n",
      "   saved : five-fold-results/final_model.joblib\n"
     ]
    }
   ],
   "source": [
    "# Cell F1: FINAL MODEL — trained on ALL training data (NO split)\n",
    "# ==================================================================\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from cuml.svm import SVC as CumlSVC\n",
    "from joblib import dump\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# ---- stream the WHOLE training file (no ID filtering) ----\n",
    "full_parts = [c for c in stream_csv(DATA_TRAIN)]\n",
    "full_df    = pd.concat(full_parts, ignore_index=True)\n",
    "\n",
    "X_full, y_full = build_xy(full_df, feat_cols)\n",
    "\n",
    "X_full = np.ascontiguousarray(X_full, dtype=np.float32)\n",
    "y_full = y_full.astype(np.int32)\n",
    "\n",
    "# ---- scale on ALL data ----\n",
    "scaler = StandardScaler()\n",
    "Xs_full = scaler.fit_transform(X_full)\n",
    "\n",
    "# ---- same hyper-parameters locked by CV ----\n",
    "final_clf = CumlSVC(\n",
    "    C=SVM_C,              # ← 必须是 CV 里定死的那个值\n",
    "    gamma=SVM_GAMMA,\n",
    "    kernel=\"rbf\",\n",
    "    #class_weight=SVM_CLASS_WEIGHT,\n",
    "    probability=True,\n",
    "    random_state=SEED,\n",
    "    tol=1e-3,\n",
    ")\n",
    "final_clf.fit(Xs_full, y_full)\n",
    "\n",
    "# ---- wrap exactly like during CV (same predict_proba interface) ----\n",
    "class _CumlPipeline:\n",
    "    def __init__(self, scaler, model):\n",
    "        self.scaler, self.model = scaler, model\n",
    "    def predict_proba(self, X):\n",
    "        return self.model.predict_proba(self.scaler.transform(X))\n",
    "    def predict(self, X):\n",
    "        return self.model.predict(self.scaler.transform(X))\n",
    "\n",
    "final_model = _CumlPipeline(scaler, final_clf)\n",
    "dump(final_model, OUT_DIR / \"final_model.joblib\", compress=3)\n",
    "\n",
    "print(\"✔ final model trained on ALL\", len(y_full), \"samples\")\n",
    "print(\"   saved :\", OUT_DIR / \"final_model.joblib\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "40cdb065-76b5-4a09-a0ff-e69c81023fd4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==============================================================\n",
      "FINAL MODEL — Independent Test Evaluation\n",
      "(trained on 100% training data, threshold = 0.5 )\n",
      "==============================================================\n",
      "  samples      : 15211\n",
      "  positives    : 8683\n",
      "  negatives    : 6528\n",
      "--------------------------------------------------------------\n",
      "  ACC      : 0.8673\n",
      "  MCC      : 0.7283\n",
      "  Sn (Rec) : 0.8966\n",
      "  Sp       : 0.8283\n",
      "  AUC-ROC  : 0.9424\n",
      "  PRC-AUC  : 0.9544\n",
      "  Prec     : 0.8741\n",
      "  F1       : 0.8852\n",
      "==============================================================\n",
      "✔ per-sample predictions saved : five-fold-results/final_model_independent_test.csv\n",
      "✔ metrics saved               : five-fold-results/final_model_metrics.csv\n"
     ]
    }
   ],
   "source": [
    "# ==================================================================\n",
    "# Cell F2: FINAL MODEL — INDEPENDENT TEST EVALUATION (8 metrics)\n",
    "# ==================================================================\n",
    "#\n",
    "# Contract\n",
    "# --------\n",
    "# - final_model was trained on 100% of the training CSV (NO hold-out).\n",
    "# - test_df is the INDEPENDENT test set and was NEVER used for:\n",
    "#       training, validation, hyperparameter tuning or threshold selection.\n",
    "# - This cell evaluates it ONCE and is READ-ONLY w.r.t. the test set.\n",
    "#\n",
    "# Output\n",
    "# ------\n",
    "# - probabilities and binary predictions (threshold = THRESHOLD)\n",
    "# - 8 metrics: ACC, MCC, Sn, Sp, AUC_ROC, PRC_AUC, Precision, F1\n",
    "# - a CSV snapshot for auditability\n",
    "# ==================================================================\n",
    "\n",
    "from pathlib import Path\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# ---- guard: objects must exist ----\n",
    "assert \"final_model\" in globals(), \"final_model missing: run the final-training cell first\"\n",
    "assert \"test_df\" in globals(), \"test_df missing: rerun Cell T5/T2\"\n",
    "assert \"feat_cols\" in globals(), \"feat_cols missing: rerun Cell T2\"\n",
    "\n",
    "# ------------------- extract X/y from test set ------------------- #\n",
    "X_test, y_test = build_xy(test_df, feat_cols)\n",
    "X_test = np.ascontiguousarray(X_test, dtype=np.float32)\n",
    "\n",
    "# ------------------- predict (ONE pass, no retraining) ---------- #\n",
    "proba = final_model.predict_proba(X_test)[:, 1]\n",
    "pred = (proba >= THRESHOLD).astype(np.int32)\n",
    "\n",
    "# ------------------- 8 metrics ---------------------------------- #\n",
    "final_metrics = evaluate(y_test, proba, threshold=THRESHOLD)\n",
    "\n",
    "# ========================== print =================================\n",
    "print(\"=\" * 62)\n",
    "print(\"FINAL MODEL — Independent Test Evaluation\")\n",
    "print(\"(trained on 100% training data, threshold =\", THRESHOLD, \")\")\n",
    "print(\"=\" * 62)\n",
    "print(f\"  samples      : {len(y_test)}\")\n",
    "print(f\"  positives    : {int((y_test == 1).sum())}\")\n",
    "print(f\"  negatives    : {int((y_test == 0).sum())}\")\n",
    "print(\"-\" * 62)\n",
    "print(f\"  ACC      : {final_metrics['ACC']:.4f}\")\n",
    "print(f\"  MCC      : {final_metrics['MCC']:.4f}\")\n",
    "print(f\"  Sn (Rec) : {final_metrics['Sn']:.4f}\")\n",
    "print(f\"  Sp       : {final_metrics['Sp']:.4f}\")\n",
    "print(f\"  AUC-ROC  : {final_metrics['AUC_ROC']:.4f}\")\n",
    "print(f\"  PRC-AUC  : {final_metrics['PRC_AUC']:.4f}\")\n",
    "print(f\"  Prec     : {final_metrics['Precision']:.4f}\")\n",
    "print(f\"  F1       : {final_metrics['F1']:.4f}\")\n",
    "print(\"=\" * 62)\n",
    "\n",
    "# ========================== audit CSV ============================\n",
    "final_out = OUT_DIR / \"final_model_independent_test.csv\"\n",
    "\n",
    "audit_df = pd.DataFrame({\n",
    "    \"ID\": _id_series(test_df).to_numpy(),\n",
    "    \"label_true\": y_test,\n",
    "    \"P_class1\": proba,\n",
    "    \"label_pred\": pred,\n",
    "})\n",
    "\n",
    "audit_df.to_csv(final_out, index=False)\n",
    "print(\"✔ per-sample predictions saved :\", final_out)\n",
    "\n",
    "# ========================== metric CSV ===========================\n",
    "metric_row = {\"model\": \"final_all_train\", \"threshold\": THRESHOLD, **final_metrics}\n",
    "final_metric_df = pd.DataFrame([metric_row])\n",
    "final_metric_df.to_csv(OUT_DIR / \"final_model_metrics.csv\", index=False)\n",
    "print(\"✔ metrics saved               :\", OUT_DIR / \"final_model_metrics.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "896e5589-e0e0-45e7-b4a6-4f25f624dd7d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "metric     CV_test_mean       FINAL      diff\n",
      "------------------------------------------------\n",
      "ACC              0.8647      0.8673   +0.0025\n",
      "MCC              0.7231      0.7283   +0.0052\n",
      "Sn               0.8945      0.8966   +0.0021\n",
      "Sp               0.8252      0.8283   +0.0031\n",
      "AUC_ROC          0.9399      0.9424   +0.0025\n",
      "PRC_AUC          0.9523      0.9544   +0.0021\n",
      "Precision        0.8719      0.8741   +0.0022\n",
      "F1               0.8830      0.8852   +0.0022\n"
     ]
    }
   ],
   "source": [
    "# ==================================================================\n",
    "# Cell F3: CV vs FINAL — sanity check\n",
    "# ==================================================================\n",
    "cv_summary = pd.read_csv(OUT_DIR / \"summary_mean.csv\")   # from Cell O1/T6\n",
    "fin = pd.read_csv(OUT_DIR / \"final_model_metrics.csv\")\n",
    "\n",
    "print(f\"{'metric':<9}{'CV_test_mean':>14}{'FINAL':>12}{'diff':>10}\")\n",
    "print(\"-\" * 48)\n",
    "\n",
    "for m in METRIC_COLS:\n",
    "    cvv = cv_summary.set_index(\"metric\").loc[m, \"TEST_mean\"]\n",
    "    fv  = fin.iloc[0][m]\n",
    "    print(f\"{m:<9}{cvv:>14.4f}{fv:>12.4f}{fv - cvv:>+10.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19d9bb3a-e2cb-46f2-a346-695724f7d6b8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "896d4f8c-282b-4ad2-b926-6416329aed6d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69556f50-3e19-41f5-8eec-f55273917d56",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:rapids-25.02_tfpt]",
   "language": "python",
   "name": "conda-env-rapids-25.02_tfpt-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
