{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ``.sig_test.split_test`` in MNIST dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.datasets import mnist\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D\n",
    "from tensorflow.python.keras import backend as K\n",
    "import time\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow.keras.optimizers import Adam, SGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(0)\n",
    "num_classes = 2\n",
    "# input image dimensions\n",
    "img_rows, img_cols = 28, 28\n",
    "\n",
    "# the data, split between train and test sets\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "X = np.vstack((x_train, x_test))\n",
    "y = np.hstack((y_train, y_test))\n",
    "ind = (y == 9) + (y == 7)\n",
    "X, y = X[ind], y[ind]\n",
    "X = X.astype('float32')\n",
    "X += .01*abs(np.random.randn(14251, 28, 28))\n",
    "y[y==7], y[y==9] = 0, 1\n",
    "\n",
    "if K.image_data_format() == 'channels_first':\n",
    "\tX = X.reshape(x.shape[0], 1, img_rows, img_cols)\n",
    "\tinput_shape = (1, img_rows, img_cols)\n",
    "else:\n",
    "\tX = X.reshape(X.shape[0], img_rows, img_cols, 1)\n",
    "\tinput_shape = (img_rows, img_cols, 1)\n",
    "\n",
    "X /= 255.\n",
    "\n",
    "# convert class vectors to binary class matrices\n",
    "y = keras.utils.to_categorical(y, num_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-06-29 16:58:55.239237: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:936] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-06-29 16:58:55.243474: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:936] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-06-29 16:58:55.243767: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:936] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-06-29 16:58:55.244460: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2022-06-29 16:58:55.245170: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:936] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-06-29 16:58:55.245475: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:936] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-06-29 16:58:55.245750: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:936] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-06-29 16:58:55.623115: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:936] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-06-29 16:58:55.623425: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:936] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-06-29 16:58:55.623674: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:936] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-06-29 16:58:55.623925: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1525] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 3763 MB memory:  -> device: 0, name: NVIDIA GeForce RTX 2060, pci bus id: 0000:01:00.0, compute capability: 7.5\n"
     ]
    }
   ],
   "source": [
    "## define the learning models\n",
    "def cnn():\n",
    "\tmodel = Sequential()\n",
    "\tmodel.add(Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape))\n",
    "\tmodel.add(Conv2D(64, (3, 3), activation='relu'))\n",
    "\tmodel.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "\tmodel.add(Dropout(0.25))\n",
    "\tmodel.add(Flatten())\n",
    "\tmodel.add(Dense(128, activation='relu'))\n",
    "\tmodel.add(Dropout(0.5))\n",
    "\tmodel.add(Dense(num_classes, activation='softmax'))\n",
    "\tmodel.compile(loss=keras.losses.binary_crossentropy, optimizer=keras.optimizers.Adam(0.0005), metrics=['accuracy'])\n",
    "\treturn model\n",
    "\n",
    "model_null, model_alter = cnn(), cnn()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "## fitting param\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "es = EarlyStopping(monitor='val_accuracy', mode='max', verbose=0, patience=15, restore_best_weights=True)\n",
    "\n",
    "fit_params = {'callbacks': [es],\n",
    "            'epochs': 50,\n",
    "            'batch_size': 32,\n",
    "            'validation_split': .2,\n",
    "            'verbose': 0}\n",
    "\n",
    "## testing params\n",
    "test_params = { 'split': \"one-split\",\n",
    "                'inf_ratio': None,\n",
    "                'perturb': None,\n",
    "                'cv_num': 2,\n",
    "                'cp': 'hommel',\n",
    "                'verbose': 2}\n",
    "\n",
    "## tuning params\n",
    "tune_params = { 'num_perm': 100,\n",
    "                'ratio_grid': [.2, .4, .6, .8],\n",
    "                'if_reverse': 0,\n",
    "                'perturb_range': 2.**np.arange(-3,3,.1),\n",
    "                'tune_ratio_method': 'fuse',\n",
    "                'tune_pb_method': 'fuse',\n",
    "                'cv_num': 2,\n",
    "                'cp': 'hommel',\n",
    "                'verbose': 2}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: ./saved/split_test/06-29_16-59/model_null_init/assets\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-06-29 16:59:00.569380: W tensorflow/python/util/util.cc:368] Sets are not currently considered sequences, but this may change in the future, so consider avoiding using them.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: ./saved/split_test/06-29_16-59/model_alter_init/assets\n",
      "====================== one-split for 0-th Hypothesis =======================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-06-29 16:59:02.617064: I tensorflow/stream_executor/cuda/cuda_dnn.cc:368] Loaded cuDNN version 8101\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(tuneHP: ratio) Est. Type 1 error: 0.000; inf sample ratio: 0.200\n",
      "✅ (tuneHP: ratio) Done with inf sample ratio: 0.200\n",
      "(tuneHP: pb) Est. Type 1 error: 0.020; perturbation level: 0.125\n",
      "✅ (tuneHP: pb) Done with inf pb level: 0.125\n",
      "cv: 0; p_value: 0.26510; loss_null: 0.00140(0.03744); loss_alter: 0.00175(0.04185)\n",
      "cv: 1; p_value: 0.68497; loss_null: 0.00211(0.04583); loss_alter: 0.00175(0.04185)\n",
      " 🧪 0-th Hypothesis: accept H0 with p_value: 0.795\n",
      "====================== one-split for 1-th Hypothesis =======================\n",
      "(tuneHP: ratio) Est. Type 1 error: 0.000; inf sample ratio: 0.200\n",
      "✅ (tuneHP: ratio) Done with inf sample ratio: 0.200\n",
      "(tuneHP: pb) Est. Type 1 error: 0.000; perturbation level: 0.125\n",
      "✅ (tuneHP: pb) Done with inf pb level: 0.125\n",
      "cv: 0; p_value: 0.16398; loss_null: 0.00140(0.03744); loss_alter: 0.00211(0.04583)\n",
      "cv: 1; p_value: 0.92602; loss_null: 0.00316(0.05611); loss_alter: 0.00175(0.04185)\n",
      " 🧪 1-th Hypothesis: accept H0 with p_value: 0.492\n",
      "====================== one-split for 2-th Hypothesis =======================\n",
      "(tuneHP: ratio) Est. Type 1 error: 0.020; inf sample ratio: 0.200\n",
      "✅ (tuneHP: ratio) Done with inf sample ratio: 0.200\n",
      "(tuneHP: pb) Est. Type 1 error: 0.100; perturbation level: 0.125\n",
      "(tuneHP: pb) Est. Type 1 error: 0.110; perturbation level: 0.134\n",
      "(tuneHP: pb) Est. Type 1 error: 0.130; perturbation level: 0.144\n",
      "(tuneHP: pb) Est. Type 1 error: 0.100; perturbation level: 0.154\n",
      "(tuneHP: pb) Est. Type 1 error: 0.090; perturbation level: 0.165\n",
      "(tuneHP: pb) Est. Type 1 error: 0.120; perturbation level: 0.177\n",
      "(tuneHP: pb) Est. Type 1 error: 0.120; perturbation level: 0.189\n",
      "(tuneHP: pb) Est. Type 1 error: 0.120; perturbation level: 0.203\n",
      "(tuneHP: pb) Est. Type 1 error: 0.110; perturbation level: 0.218\n",
      "(tuneHP: pb) Est. Type 1 error: 0.110; perturbation level: 0.233\n",
      "(tuneHP: pb) Est. Type 1 error: 0.150; perturbation level: 0.250\n",
      "(tuneHP: pb) Est. Type 1 error: 0.130; perturbation level: 0.268\n",
      "(tuneHP: pb) Est. Type 1 error: 0.100; perturbation level: 0.287\n",
      "(tuneHP: pb) Est. Type 1 error: 0.120; perturbation level: 0.308\n",
      "(tuneHP: pb) Est. Type 1 error: 0.090; perturbation level: 0.330\n",
      "(tuneHP: pb) Est. Type 1 error: 0.060; perturbation level: 0.354\n",
      "(tuneHP: pb) Est. Type 1 error: 0.070; perturbation level: 0.379\n",
      "(tuneHP: pb) Est. Type 1 error: 0.040; perturbation level: 0.406\n",
      "✅ (tuneHP: pb) Done with inf pb level: 0.406\n",
      "cv: 0; p_value: 0.00000; loss_null: 0.00246(0.04950); loss_alter: 0.03088(0.17298)\n",
      "cv: 1; p_value: 0.00000; loss_null: 0.00175(0.04185); loss_alter: 0.03298(0.17859)\n",
      " 🧪 2-th Hypothesis: reject H0 with p_value: 0.000\n"
     ]
    }
   ],
   "source": [
    "## Inference based on dnn_inference\n",
    "from dnn_inference.sig_test import split_test\n",
    "## testing based on learning models\n",
    "inf_feats = [[np.arange(19,28), np.arange(13,20)], [np.arange(21,28), np.arange(4, 13)],[np.arange(7,16), np.arange(9,16)]]\n",
    "cue = split_test(inf_feats=inf_feats, model_null=model_null, model_alter=model_alter, eva_metric='zero-one')\n",
    "P_value = cue.testing(X, y, fit_params, test_params, tune_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAEWCAYAAAB7QRxFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAXZklEQVR4nO3deZRU1bXH8d+BZlAQEEFQAVFQJKg4m2gQVJ5DRHkqMQ5J8KkvspCoyMIgaIhjiDEOUYkaRTTRhDgkhhjzcGKKojLEKFNUQCEKiIKMMp73xy1i7brdXQ1dPezq72etXqt/t+69dbp31+7T51ZXhRijAAB+1avpAQAAKodGDgDO0cgBwDkaOQA4RyMHAOdo5ADgnOtGHkIYG0K4pabHgapBfYsb9S0c1428NgghdAwhvBpCWB9CmBdC6F3OvrNDCGuzPraEEMZn3X5mCOHdzG2vhRC+lnXbxSGErTnH96rarw47Ut+sY1qGED4NIUzN2X5ZCOH9TO3+FkLYO+u2oZnarwkhLAwhDK2KrwfWDj5+G4UQxoQQVocQloYQrsm5/eTMOdZnzrlvRY+tLBp55f1O0ixJe0gaIenpEELr0naMMXaLMTaNMTaVtJukxZKekqQQwgGSnpA0QFILSeMl/TmEUJJ1ite3H5/5mFhFXxO+UuH6ZvmZpLnZGzK/dG+T1FdSS0kLM+f+zy6Svi9pd0mnSRoUQji/8sNHHjtS359IOkDSvpJOlHRtCOE0SQohtJL0rKQblNR3uqRxFTm2IGKMBfuQtEjSdZLmSFop6VFJjfMcM1dSn6xcIulTSUdk8lOSlkr6QtJkSd2y9h0r6ZbM5xdLmppz7iipc+bzRpLukPSRpGWSHpC0S+a2VpL+ImmVpM8lTZFUrwJf74GSNkraLWvbFEkDKnBsT0lrJDXJ5EGSns+6vZ6kDZJOLuvrq+4P6pu/vpKOk/S6pP/JHm9mbPdn5b0z4+9Uxnl+Kele6lt76ivpY0mnZOWbJf0+8/kPJL2WdVsTJY/fg/IdW4iPqpiRXyTpVEmdMt+o6/Ps/ztJF2TlUyWtiDHOzOQXlPwm21PSTCWz1p0xKjOewyR1lrSPpB9nbhsiaYmk1pLaSBqu5IdIIYTRIYTRZZyzm6QFMcY1WdvezmzPp7+kZ2KM67K2hZzPg6SDs7YdHkJYEUL4VwjhhpzZenWhvmXUN4RQX9J9Sn4pl/baF7n1lWx9t58nSOohaXYZ46pK1LeU+oYQdpe0V+b20vbtln1b5nH9gaRuFTi20qqikd8XY1wcY/xc0q2yRS7Nk5LOCiHsmskXKutPzhjjmBjjmhjjRiV/nnQPITTfkQFlHhg/kDQ4xvh5pnC3Sdr+p+tmJd/ofWOMm2OMU+L2KUGMA2OMA8s4dVMlM41sXyhZNilvPLtK6qdkRrLdS5J6hhB6hRAaKvlhbChp+/dlspIH/Z6SzlXyfa2JdVTqW3Z9r5T0RoxxRim3/U3SeSGEQ0MIuyhpQlFf1TfbT5Q8Nh8t436qEvUtvb5Ns24vbd/yzpXv2Eqrika+OOvzD5X8CVmmGOP7Sv48OzPzw3CWkh8OhRDqhxBGhRA+CCGsVvKnn5T8KbUjWit5wMwIIawKIaxS8sDavhb2c0nvS5oQQlgQQhhWwfOuldQsZ1szJUsm5TlHyZ+Ak7ZviDHOUzJLv0/SJ0q+xjlKZhqKMS6IMS6MMW6LMb4j6SYlvwyqG/Utpb6ZC5dXKllnTYkxviRppKRnlHydizLnWZJznkFK1srPyDS/6kZ9S3/8rs26vbR9yztXvmMrrSoaefuszzsoWRvKZ/ufZ30lzcn8cEjJb/e+knpLai6pY2Z7yD2BpHXKmt2EENpm3bZCyXpVtxhji8xH85hcdFRmxjAkxri/kh/Ea0IIJ1dg3LMl7R9CyP7N2l35/yTuL+nx7bOG7WKMT8cYD44x7qHkQd9R0ltlnCOq9O9DVaO+pdf3GCWzwjkhhKWS7pF0TOYZCvUz47g/xnhAjLGNkoZeIundrK/pEknDlFwXWZK6h+pBfUupb4xxpZIJVvcy9p2dfVsIoYmS5anZFTi28gq12B6/uljyjqR2Sq7cTpV0WwWO20vSeiXLB1dlbR8o6R9Kfns1kTRa9gLIWH11sWT7hYvDJDVWcjEke997JP1B0p6ZvI+kUzOf91Gy7haU/CB/IunECn7N05RchGks6WwlF1xal7N/O0lbVMpFLklHSqqvZKbxB0lPZt12uqQ2mc8PUtIARhayftR35+ur5GJc26yPqyS9Ialt5vbGSpbGgpIGOTH7e6dkbXqppK7VWVPqW/HHr5J1+klKnll0UOZ+Tsvc1lrJcsm5mXP9TNK0ihxbkNpVwQ/C9qveqyQ9JmnXCh77spIG1zZrW1NJzyn5E+RDJX9ylvqDkMkjlPz2Xizpuzn7NlayrrZA0molfw5embltcGbs65T8qXtD1jkfkPRAOePuqORBuUHSfEm9cx6cs3P2v07SlDLONTXztX4u6UFlntGSue0OJVfr12W+hpskNaiqBzX13bn6Zt12seyzVlpI+mdmDEsl/VRS/azbFypZ612b9VHmuKhv9ddXyS/rMZn7Xybpmpxz9ZY0L3OuiZI6VvTYyn6EzJ0URAhhkaTLYrIeiCJDfYsb9fWLfwgCAOeqpZGHEIYH+6/l2z9eqI77R9WivsWN+tZ+BV1aAQBUP5ZWAMA5GjkAOFfua3UMCB0frK6BoHo8EBddvv1z6lt8qG9xy65vNmbkAOAcjRwAnKORA4BzNHIAcI5GDgDO0cgBwDkaOQA4RyMHAOdo5ADgHI0cAJyjkQOAczRyAHCORg4AztHIAcA5GjkAOEcjBwDnaOQA4ByNHACco5EDgHM0cgBwjkYOAM7RyAHAORo5ADhHIwcA52jkAOAcjRwAnKORA4BzJTU9gNrql8smmhwbNjF58vJocp8Lhpsc6tVPnfOwvv1svuF/KzFCVAb1rVE9qvoOup50rMkDnrlvSnYutvoyIwcA52jkAOAcjRwAnKsTa+Q3/OQUk9tcfIXJT69slTqmRd9RJm/bssnk3DW0eiUN847jnReet+e88SGTjxj5g7znQBr1LW6dOtv6NWrXweSlm9O1eWnmApN/9F/Xm1xs9WVGDgDO0cgBwDkaOQA4RyMHAOdq/cXO09s0SW37271PmPzy5EXlnuOZGe+ZvPavD5oct21NHXPSJf1NfnXsb8u9j9xzlPYPBVs3bTB59YoNqX3qGupb3Eqr7wt7dTH5s8/L/z4tXb3O5C2zFtsdov3nHknao/0+JpdWL3MK5/VlRg4AztHIAcA5GjkAOFfr1shv/sXZJvdccmJqnwW3/8rkfOtf+byy65zUtvkjLjH5+kNam7zfqYeafNLyb5m86PX/S50zd5zf69vV5MX35h+rd9S3uFWkvvs/cpPdEEKl7vOY+mtT29bNf8fkl4/73ORiqy8zcgBwjkYOAM7RyAHAuVq3Rn53V/vCMwvHpReecteqcp8D2nj3tiaveNi+IPwHt4w0+a5f5zwvtRRvTLb7XDnuLyYv6ntduWOSpANPOsvk9uf1NTn/KPyjvsWtIvXdP3dNPOd54PUaNDK59yG23uvft/83sGjJl3nH9XiR15cZOQA4RyMHAOdo5ADgXI2vkd9677dN7nL/c3mPyV2/6tyzj8l/qfeUyYP2PXMnR1e2VxbY56VW5LUanv9RT5Nvvb32vFZDVaG+xW1n6pu7Jr5ryz1NPip8YvKciTN2bnDlKLb6MiMHAOdo5ADgHI0cAJyr8TXyew641OS1j91lcmlrVRdec7nJP/33YyaPHDq50uM6okVjk/s/Z5+b3P72P5Z7/AVXX5ba1nDMiEqPyxvqW9x2pr5779fe5C5f/tvk9+avrPS4mpXYOerNL99qcrHVlxk5ADhHIwcA52jkAOBcja+RX7fpJZOnfu+7Jh/SvnnqmJ4DLjB55PJ1qX0qq//CKSa3OHVkGXsmjj7vfJP7XP291D5VMc7ajvoWt52p727n29csf29T+nVNKmufXl83ucXQKWXsmfBeX2bkAOAcjRwAnKORA4BzNHIAcK7GL3YOPusOkw/MuX1jKcdMKPAYfvFk+sn/rc++s9xjSho3NXnit1uYfMW1tffCSHWivsVtZ+pb2rbKOKh7u9S2l2cutBs6H25isdWXGTkAOEcjBwDnaOQA4FyNr5HXBnvcmX7b1NJe7CfbF4+cY/LA/c4uY0/UNOpb65T/3zk7aFiDbqltoXPdqi8zcgBwjkYOAM7RyAHAuTq5Rn7b6jkmjzlhUGqfejnfmdw3Z5126dCCjwuFQX2LG/VNY0YOAM7RyAHAORo5ADhXJ9bId29gf1/96s0lJtcraZA6Jvd5xmufHmjy4P36Fmh0qCzqW9yob37MyAHAORo5ADhHIwcA54pyjbz9LvbLuu6fvzX5rKMu6pGde8eYOseR3Q+w5+hsX4thc/qQiijoa0zUVfnq2+Siu/Oe40+P2zdb/nX3E0zeyfqiAKjvjmNGDgDO0cgBwDkaOQA4RyMHAOeK4mLngU0bmtzhtYkmN7noFpNPzjn+lCM7ps4599XXTS6yayOu7Gh9c63949WpbYM7HGtysV388oT6Vh4zcgBwjkYOAM7RyAHAuaJYIz/u2L1N7nXJjSbnvoBO01Z7mjzn1elVM7C6oUf+XSpnXSNbv7HfPMvkk0Mw+eMf3mz+8WpQ+9OraGQohB19/HY9xdaf+jIjBwD3aOQA4ByNHACcc7lG3r/XviY3ePSPdodzhpl4yBn/bXKX268weVnBRoZC2KdlY5PDIUfYHWbON3G3PduYPOqpESY/W7ihoQAq+/i95cnrTKa+zMgBwD0aOQA4RyMHAOdcrpF/dv84k7/db0QZeyZe/35rk68YubngY0LhbOp2uMmzZv2r3P2/sbd9rY5nJ31W8DGhcCr/+KW+uZiRA4BzNHIAcI5GDgDO1fo18ktP75Ta9trK9eUeM3P87Sa/8Z0+BR0TCqddq11T21Zu3lruMccf1cXkVbNmFHRMKBwev9WDGTkAOEcjBwDnaOQA4FytXyPv8Ls/p7adcMpQkzv3/JbJq3LW1B6fvLjwA0NB7HLYkalt096aZ3KTlvZ5xJtz1sQ/Xvll4QeGguDxWz2YkQOAczRyAHCORg4AztHIAcC5Wn+xU78enneXaUfPNnnIz7k44sbi+Xl3+UbztSbPW8jFTTd4/FYLZuQA4ByNHACco5EDgHO1fo185NDnUtv65+Qh51XPWFB4781fntrWLifP+0e1DAVVgMdv9WBGDgDO0cgBwDkaOQA4F2KMNT0GAEAlMCMHAOdo5ADgHI0cAJyjkQOAczRyAHCORg4AztHIAcA5GjkAOEcjBwDnaOQA4ByNHACco5EDgHM0cgBwjkYOAM6V+1ZvA0LHB6trIKgeD8RFl2//nPoWH+pb3LLrm40ZOQA4RyMHAOdo5ADgHI0cAJyjkQOAczRyAHCORg4AztHIAcA5GjkAOEcjBwDnaOQA4ByNHACco5EDgHM0cgBwjkYOAM7RyAHAORo5ADhHIwcA52jkAOAcjRwAnKORA4BzNHIAcI5GDgDO0cgBwDkaOQA4RyMHAOdo5ADgHI0cAJwrqekBVIchA48xeduPHzb50rHTU8dMf3qcyXHbVpPXPj/M5K3vTjH5w0POTZ2z3k2XmfyL0W+WMWLsCOrrTo8d2bljh+Z2Q+dDTHzn31+kjvnik09MfrHzYaaAxVZfZuQA4ByNHACco5EDgHNFuUZ+94sjTe7yREOTl/UZmvccA68fZPKb8z41uVnfO8s9Pm6bltr22zF32w2jj8s7DqRR3+LW9ejOJk/+2M43N06fl/ccHTrva/LR511ocrHVlxk5ADhHIwcA52jkAOAcjRwAnHN5sbNJ/WDyFT/8hsnt7l5m8tplC02+fPgVJg+b+2DqPu7q19fkvo3qm9xn/F9NvnHYXSbn/oOJJJ3VdovJL6X2gER9i139nNyhYwuTX1m00eStGzfY/TvZC5n7r/sodR+LZs0wue/8y00utvoyIwcA52jkAOAcjRwAnHO5Rj7qrdEmNxswPmcPu2Z2w6ghJjc9u4/JN67ZlPc+b5l0u8nNB99Vxp6J/Y4/M7Xtqa6n5r0fUN9i1+X4biZPeHd5zh5fmtS5y34m159pXwTt/a0x731eNeGnJhdbfZmRA4BzNHIAcI5GDgDO1fo18p+tmZPattuJg00O9ewzU98ZP8rk5zsfZfLcCqyZ5jp+QrMd2n9Gj/QL+wy5d0Mpe9Zt1Le4lVbfi/Y5wW4I9v8GehzZxeTlk/5u8roKrInnKvb6MiMHAOdo5ADgHI0cAJyrdWvkd6+bbXKLnoNT+zRs2tLkn4+yr6Pw5L5HmPzxl/Y1EnK1aZT76g/StY/ZN1rd5xG7Tpf7WgyfTbrb5CFNupZ7n3UV9S1uFanvKSUNTD6oSweTP55oa7FxW/lr4g1LmY7uf2h7k3//WnHXlxk5ADhHIwcA52jkAOBcja+Rd2/eyOQuV/zJ5NznEEvpNdNZX++1Q/d5wdF7mXzHpek3Ym11529Mzl1Te/zhH5t8b+tDd2gMdQX1LW47U9/cNfHVr7+xQ/e5V859Lmx3UGqflxZ+bPKXJUtNLrb6MiMHAOdo5ADgHI0cAJyr8TXy3RvYNbTlc/5exp5f6fjdc0xu3mpXk8+Z/4rJ/ccvNrnfMxNNXv3QI6n7qFfSsNx8xpxHTR6yfnPZA67DqG9x25n67vL2TJNLGtj5ZJsTjjX5n8vt65PPWvaZyVsWL0nfSbDnLPb6MiMHAOdo5ADgHI0cAJyjkQOAczV+sXPlZvuPGG0Oti86v+zdyalj+rY42uTQ0l5wufjUkfb2nH9KaH/MKSa3aN85dR9Lpr9o8p5fO97kIRcOSx2DNOpb3HamvjNKmtsNDewbS+it92zOeeOJXZq3MrmkcZPUfXz5xQqTi72+zMgBwDkaOQA4RyMHAOdqfI387S82mvxi42dN7tXZrndK0op500z+2mlnm/zXIT1M/ug7Z5q85A+T7H3e90TqPsZOt3ni7WeYPGpMca2xVRXqW9x2pr4vbtk0JTt3+1Y/c3ve+v7jI3u+0up7x0Mmzy3y+jIjBwDnaOQA4ByNHACcCzGW/camA0LHB6txLNVmxA29Te40fltqn3o5bxC79tkrTR7Uwa65efFAXPSfd22gvl+hvn5Q3zRm5ADgHI0cAJyjkQOAczX+PPKa0GSvPUyuV7Iq7zHjDu+Xdx/UDtS3uFHfNGbkAOAcjRwAnKORA4BzdXKNfPjAcebFHHo326OUvexrIL+7YWMp++ywKfl3QWUNHzjO5HhE79Q+ua9hDj+obxozcgBwjkYOAM7RyAHAORo5ADhXJy92HtitrclLFm9J75Tzhq/w47bR3zF57JhVqX3q2sWwYkJ905iRA4BzNHIAcI5GDgDO1ck18pJWe9sNiz8sZS/WyCuoR/5dqlfz0883/3gVH74/tU9dW0MtJs1PP99k6suMHADco5EDgHM0cgBwrk6uka+ZO8/kGBun9gkq+02pUbu9ffW1Jsdt7VP7xG1bq2s4KDDqm8aMHACco5EDgHM0cgBwrk6ukS9evt7k3Trtl9pn7YrlJrfYu5nJKz9cVfBxoTAeeu49k7sNvya1z7yXxpt87IXdTZ5y77TCDwwFQX3TmJEDgHM0cgBwjkYOAM7VyTXyXMe1TW+b8Kl9HvmHe3Qyud26OSYvWbGh4ONCYbzZL71ttwn2ecajTx5h8iXzrzJ5zIQFBR8XCoP6MiMHAPdo5ADgHI0cAJxjjVzSh9PmpLZ1OuIokz/41yKTWxxsn5e6y+dvmLxhG6/VUluM7nlVatt1z9jnGY+6/h6Tj3nwNyZ3nNrL5EXrNxdmcKg06suMHADco5EDgHM0cgBwjkYOAM5xsVPSuq3pC5NNZk43+Zs9jzd56oz5Jn+9pX1zCv5BqPaYu2ZTalvXc880+e33bb27nznM5EnftG9e4P0fSIoJ9WVGDgDu0cgBwDkaOQA4xxp5GXLXzde9MtXkdjn7L6ni8aCwctdV57Y51OT+OfuPqeLxoLDqWn2ZkQOAczRyAHCORg4AzoUYeXEnAPCMGTkAOEcjBwDnaOQA4ByNHACco5EDgHM0cgBw7v8BR7oW7PR3O+8AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 6 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P-values: [0.7953140193691346, 0.49192858264606976, 1.1876805112986783e-19]\n"
     ]
    }
   ],
   "source": [
    "## visualize testing results\n",
    "cue.visual(X,y)\n",
    "print('P-values: %s' %P_value)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf",
   "language": "python",
   "name": "tf"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}