Merge remote-tracking branch 'refs/remotes/origin/main'

Main.ipynb

@@ -15,7 +15,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 1,
    "id": "78fdf40a",
    "metadata": {
     "ExecuteTime": {
@@ -34,8 +34,10 @@
     "import numpy as np\n",
     "import pandas as pd\n",
     "import os\n",
+    "import hashlib\n",
     "from IPython.display import display\n",
-    "from scipy import stats\n"
+    "from scipy import stats\n",
+    "\n"
    ]
   },
   {
@@ -63,7 +65,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Configs ok: {'KPvKP', 'KBNvK', 'KRvKB', 'KQvK', 'KNvK', 'KRvK', 'KBvK', 'KPvK', 'KRBvKP', 'KRPvKP', 'KQvKP'}\n"
+      "Configs ok: {'KRBvKP', 'KRvKB', 'KRvK', 'KBNvK', 'KBvK', 'KRPvKP', 'KPvK', 'KNvK', 'KQvK', 'KQvKP', 'KPvKP'}\n"
      ]
     },
     {
@@ -100,7 +102,7 @@
       "Récupération des positions de la tablebase KRvKB\n",
       "Récupération des positions de la tablebase KBvK\n",
       "Récupération des positions de la tablebase KNvK\n",
-      "Total positions récupérées: 110000\n",
+      "Total positions récupérées: 55000\n",
       "Ajoute de toutes les positions des tablebases dans Redis terminé!\n"
      ]
     }
@@ -110,7 +112,7 @@
     "    positions = []\n",
     "    tries = 0\n",
     "\n",
-    "    while len(positions) < 10000 and tries < 10000000:\n",
+    "    while len(positions) < 10000 and tries < 5000:\n",
     "        board = generate_board_from_config(config)\n",
     "        positions.append(board)\n",
     "        tries += 1\n",
@@ -289,11 +291,11 @@
      "text": [
       "BENCHMARK SYZYGY\n",
       "BENCHMARK REDIS\n",
-      "Redis latency array length: 110000\n",
+      "Redis latency array length: 55000\n",
-      "Syzygy latency array length: 100273\n",
+      "Syzygy latency array length: 50127\n",
-      "            test   statistic  p-value  \\\n",
+      "            test  statistic  p-value  \\\n",
-      "0  paired t-test  108.613717      0.0   \n",
+      "0  paired t-test  85.042506      0.0   \n",
-      "1        KS-test    0.508073      0.0   \n",
+      "1        KS-test   0.546193      0.0   \n",
       "\n",
       "                                    interpretation  \n",
       "0         H0: mean latency equal (Redis vs Syzygy)  \n",
@@ -567,29 +569,10 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "id": "a68060e1-0052-4446-a0d3-40bc1756f210",
    "metadata": {},
-   "outputs": [
+   "outputs": [],
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "MEMORY USAGE REDIS\n",
-      "Memory Redis: 178.58M\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "'\\n#Test Mémoire Syzygy\\nimport psutil\\n\\ntablebases = chess.syzygy.open_tablebase(\"tablebases/\")\\n\\n#Récuperer les données de proccesus de Syzygy\\nprocess = psutil.Process(os.getpid())\\nmem_bytes = process.memory_info().rss\\nmem_mb = mem_bytes / (1024**2)\\nprint(\"MEMORY USAGE SYZYGY\")\\nprint(f\"Memory Syzygy: {mem_mb}\")\\n'"
-      ]
-     },
-     "execution_count": 5,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
    "source": [
     "#Test Mémoire Redis ( Stockage du Redis )\n",
     "redis_server = redis.Redis(host=\"localhost\", port=6379, db=0)\n",
@@ -612,11 +595,196 @@
     "print(f\"Memory Syzygy: {mem_mb}\")\n",
     "\"\"\""
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "46157e5c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_250_random_position(tablebases, config):\n",
+    "    positions = []\n",
+    "    tries = 0\n",
+    "\n",
+    "    while len(positions) < 250 and tries < 5000:\n",
+    "        board = generate_board_from_config(config)\n",
+    "        try:\n",
+    "            tablebases.probe_wdl(board)  # thử probe WDL\n",
+    "            positions.append(board)\n",
+    "        except chess.syzygy.MissingTableError:\n",
+    "            pass  # bỏ qua nếu table không tồn tại\n",
+    "        tries += 1\n",
+    "    return positions\n",
+    "\n",
+    "def add_tablebase_to_redis():\n",
+    "    h = hashlib.new('sha512_256')\n",
+    "    #Connect to Redis server\n",
+    "    redis_server = redis.Redis(host='localhost', port=6379, db=1) #db1 pour H4\n",
+    "    \n",
+    "    tablebases_path = \"tablebases/\" #Chemin vers les tables bases Syzygy\n",
+    "\n",
+    "    tablebases = chess.syzygy.open_tablebase(tablebases_path)\n",
+    "\n",
+    "    available_tables = set()\n",
+    "    for f in os.listdir(tablebases_path):\n",
+    "        if f.endswith(\".rtbw\") or f.endswith(\".rtbz\"):\n",
+    "            available_tables.add(f.split(\".\")[0])\n",
+    "\n",
+    "    print(\"Configs ok:\", available_tables)\n",
+    "    \n",
+    "    configs = [\"KBNvK\",\"KPvK\",\"KPvKP\",\"KQvK\",\"KQvKP\",\"KRBvKP\",\"KRPvKP\",\"KRvK\",\"KRvKB\"]\n",
+    "    config_valide = [c for c in configs if c in available_tables]\n",
+    "    #Configurations de toute les tablebases que l'on utilisera dans le projet.\n",
+    "    all_positions = []\n",
+    "    for config in config_valide:\n",
+    "        print(f\"Récupération des positions de la tablebase {config}\")\n",
+    "        position = get_250_random_position(tablebases,config)\n",
+    "        all_positions.extend(position) #Ajout des positions générées à la liste globale\n",
+    "        \n",
+    "    print(f\"Total positions récupérées: {len(all_positions)}\")\n",
+    "    for position in all_positions:\n",
+    "        try:\n",
+    "            fen = position.fen()\n",
+    "            key = hashlib.md5(fen.encode()).hexdigest()[:16] #Limite à 16 caractères\n",
+    "            wdl = tablebases.probe_wdl(position)\n",
+    "            dtz = tablebases.probe_dtz(position)\n",
+    "            redis_server.hset(key, mapping={\"wdl\": wdl, \"dtz\": dtz})  \n",
+    "        except (KeyError,chess.syzygy.MissingTableError): #vérifie que les positions générés appartiennent bien au table du projet seulement\n",
+    "            continue  \n",
+    "        \n",
+    "    print(\"Ajoute de toutes les positions  hashés des tablebases dans Redis terminé!\")\n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "        add_tablebase_to_redis()\n",
+    "    \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "099ce824",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def benchmark_hash(redis_serveur, pos):\n",
+    "    latency = []\n",
+    "    start_time_all = time.time()\n",
+    "\n",
+    "    for board in pos:\n",
+    "        fen = board.fen()\n",
+    "        key = hashlib.md5(fen.encode()).hexdigest()[:16] #Limite à 16 caractères\n",
+    "        start_individual = time.time()\n",
+    "        results = redis_serveur.hgetall(key)\n",
+    "        latency.append(time.time() - start_individual)\n",
+    "\n",
+    "    total_time_all=time.time()-start_time_all\n",
+    "    latency= np.array(latency)\n",
+    "\n",
+    "    return{\n",
+    "        \"count\": len(pos),\n",
+    "        \"average_latency\": latency.mean(),\n",
+    "        \"p50\": np.percentile(latency,50),\n",
+    "        \"p90\": np.percentile(latency,90),\n",
+    "        \"p95\": np.percentile(latency,95),\n",
+    "        \"p99\": np.percentile(latency,99),\n",
+    "        \"std\": latency.std(),\n",
+    "        \"min\": latency.min(),\n",
+    "        \"max\": latency.max(),\n",
+    "        \"total_time\": total_time_all,\n",
+    "        \"latency\": latency,\n",
+    "        \"rps\": len(latency)/total_time_all,\n",
+    "    }"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "646babb4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def statistical_tests(bench_redis, bench_hash):\n",
+    "    redis_lat = np.array(bench_redis[\"latency\"], dtype=np.float64)\n",
+    "    hash_lat = np.array(bench_hash[\"latency\"], dtype=np.float64)\n",
+    "    min_len = min(len(redis_lat), len(hash_lat))\n",
+    "    redis_lat = redis_lat[:min_len]\n",
+    "    hash_lat = hash_lat[:min_len]\n",
+    "    \n",
+    "    # PAIRED T-Test (Comparaison median)\n",
+    "    t_stat, t_p = stats.ttest_rel(hash_lat, redis_lat)\n",
+    "    \n",
+    "    # KS Test (Comparaison distribution)\n",
+    "    ks_stat, ks_p = stats.ks_2samp(hash_lat, redis_lat)\n",
+    "    \n",
+    "    df_stats = pd.DataFrame({\n",
+    "        \"test\": [\"paired t-test\", \"KS-test\"],\n",
+    "        \"statistic\": [t_stat, ks_stat],\n",
+    "        \"p-value\": [t_p, ks_p],\n",
+    "        \"interpretation\": [\n",
+    "            \"H0: mean latency equal (Redis vs Hash)\",\n",
+    "            \"H0: same latency distribution (Redis vs Hash)\"\n",
+    "        ]\n",
+    "    })\n",
+    "    print(df_stats)\n",
+    "    return df_stats\n",
+    "\n",
+    "def run_experiment():\n",
+    "    redis_server = redis.Redis(host=\"localhost\", port=6379, db=0)\n",
+    "    redis_hash = redis.Redis(host=\"localhost\", port=6379, db=1)\n",
+    "    tablebases = chess.syzygy.open_tablebase(\"tablebases/\")\n",
+    "    \n",
+    "    configs = [\"KBNvK\",\"KPvK\",\"KPvKP\",\"KQvK\",\"KQvKP\",\"KRBvKP\",\"KRPvKP\",\"KRvK\",\"KRvKB\"]\n",
+    "    \n",
+    "    all_positions = []\n",
+    "    for config in configs:\n",
+    "        all_positions += get_250_random_position(tablebases, config)\n",
+    "    \n",
+    "    # Filter to have 2 paired test have the same N\n",
+    "    valid_positions = []\n",
+    "    for board in all_positions:\n",
+    "        try:\n",
+    "            tablebases.probe_wdl(board)\n",
+    "            valid_positions.append(board)\n",
+    "        except chess.syzygy.MissingTableError:\n",
+    "            continue\n",
+    "    \n",
+    "    print(\"BENCHMARK REDIS (FEN)\")\n",
+    "    bench_redis = benchmark_redis(redis_server, valid_positions)  # ← db=0 (FEN)\n",
+    "    print(bench_redis)\n",
+    "    \n",
+    "    print(\"BENCHMARK REDIS (HASH)\")\n",
+    "    bench_hash = benchmark_hash(redis_hash, valid_positions)  # ← db=1 (Hash), fonction différente\n",
+    "    print(bench_hash)\n",
+    "    \n",
+    "    # Afficher les longueurs APRÈS avoir créé les benchmarks\n",
+    "    print(f\"Redis FEN latency array length: {len(bench_redis['latency'])}\")\n",
+    "    print(f\"Redis Hash latency array length: {len(bench_hash['latency'])}\")\n",
+    "    \n",
+    "    print(\"MEMORY USAGE REDIS (FEN - db=0)\")\n",
+    "    mem_fen_bytes, mem_fen_human = get_redis_memory(redis_server)\n",
+    "    print(f\"Memory: {mem_fen_human} ({mem_fen_bytes} bytes)\")\n",
+    "    \n",
+    "    print(\"MEMORY USAGE REDIS (HASH - db=1)\")\n",
+    "    mem_hash_bytes, mem_hash_human = get_redis_memory(redis_hash)\n",
+    "    print(f\"Memory: {mem_hash_human} ({mem_hash_bytes} bytes)\")\n",
+    "    \n",
+    "    return {\n",
+    "        \"bench_redis\": bench_redis,\n",
+    "        \"bench_hash\": bench_hash,\n",
+    "        \"memory_redis\": mem_fen_bytes,\n",
+    "        \"memory_hash\": mem_hash_bytes\n",
+    "    }\n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "    results = run_experiment()\n",
+    "    statistical_tests(results[\"bench_redis\"], results[\"bench_hash\"])"
+   ]
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": ".venv",
+   "display_name": "Python 3",
    "language": "python",
    "name": "python3"
   },
@@ -630,7 +798,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.3"
+   "version": "3.10.12"
   }
  },
  "nbformat": 4,