Getting slurm data using slurm2sql

requirements.txt

+astroid==2.3.3
+astropy==4.0
+attr==0.3.1
+attrs==19.3.0
+backcall==0.1.0
+bleach==3.1.3
+certifi==2019.11.28
+chardet==3.0.4
+confuse==1.0.0
+cycler==0.10.0
+decorator==4.4.2
+defusedxml==0.6.0
+entrypoints==0.3
+htmlmin==0.1.12
+idna==2.8
+ipykernel==5.1.4
+ipython==7.13.0
+ipython-genutils==0.2.0
+ipywidgets==7.5.1
+isort==4.3.21
+jedi==0.16.0
+Jinja2==2.11.1
+joblib==0.14.1
+jsonschema==3.2.0
+jupyter-client==6.0.0
+jupyter-core==4.6.1
+kaggle==1.5.6
+kiwisolver==1.1.0
+lazy-object-proxy==1.4.3
+llvmlite==0.31.0
+MarkupSafe==1.1.1
+matplotlib==3.2.0
+mccabe==0.6.1
+missingno==0.4.2
+mistune==0.8.4
+more-itertools==8.2.0
+nbconvert==5.6.1
+nbformat==5.0.4
+networkx==2.4
+notebook==6.0.3
+numba==0.48.0
+numpy==1.18.2
+packaging==20.3
+pandas==0.25.3
+pandas-profiling==2.5.0
+pandocfilters==1.4.2
+parso==0.6.2
+pexpect==4.8.0
+phik==0.9.9
+pickleshare==0.7.5
+pluggy==0.13.1
+prometheus-client==0.7.1
+prompt-toolkit==3.0.3
+ptyprocess==0.6.0
+py==1.8.1
+Pygments==2.6.1
+pylint==2.4.4
+pyparsing==2.4.6
+pyrsistent==0.15.7
+pytest==5.4.1
+pytest-pylint==0.15.1
+python-dateutil==2.8.1
+python-slugify==4.0.0
+pytz==2019.3
+PyYAML==5.3
+pyzmq==18.1.1
+requests==2.22.0
+scipy==1.4.1
+seaborn==0.10.0
+Send2Trash==1.5.0
+six==1.14.0
+slurm2sql==0.9.0
+tangled-up-in-unicode==0.0.3
+terminado==0.8.3
+testpath==0.4.4
+text-unidecode==1.3
+tornado==6.0.4
+tqdm==4.42.0
+traitlets==4.3.3
+urllib3==1.25.8
+visions==0.2.2
+wcwidth==0.1.8
+webencodings==0.5.1
+widgetsnbextension==3.5.1
+wrapt==1.11.2

slurm-2sql.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import sqlite3\n",
+    "import slurm2sql\n",
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "db = sqlite3.connect('test.db')\n",
+    "slurm2sql.slurm2sql(db, ['-S', '2020-03-18', '-a'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# For example, you can then convert to a dataframe:\n",
+    "df1 = pd.read_sql('SELECT * FROM slurm', db)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>JobID</th>\n",
+       "      <th>ArrayJobID</th>\n",
+       "      <th>ArrayTaskID</th>\n",
+       "      <th>JobStep</th>\n",
+       "      <th>JobIDSlurm</th>\n",
+       "      <th>JobName</th>\n",
+       "      <th>User</th>\n",
+       "      <th>Group</th>\n",
+       "      <th>Account</th>\n",
+       "      <th>State</th>\n",
+       "      <th>...</th>\n",
+       "      <th>MaxDiskReadNode</th>\n",
+       "      <th>MaxDiskReadTask</th>\n",
+       "      <th>MaxDiskWrite</th>\n",
+       "      <th>MaxDiskWriteNode</th>\n",
+       "      <th>MaxDiskWriteTask</th>\n",
+       "      <th>ReqGPUS</th>\n",
+       "      <th>Comment</th>\n",
+       "      <th>GPUMem</th>\n",
+       "      <th>GPUEff</th>\n",
+       "      <th>NGPU</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>3319116</td>\n",
+       "      <td>3319116</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>None</td>\n",
+       "      <td>3319116_[43-45,47%5]</td>\n",
+       "      <td>1mUD1MPa</td>\n",
+       "      <td>user</td>\n",
+       "      <td>user</td>\n",
+       "      <td>user</td>\n",
+       "      <td>PENDING</td>\n",
+       "      <td>...</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>NaN</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>3927198</td>\n",
+       "      <td>3887451</td>\n",
+       "      <td>30.0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>3887451_30</td>\n",
+       "      <td>100kCrC20MPa</td>\n",
+       "      <td>user</td>\n",
+       "      <td>user</td>\n",
+       "      <td>user</td>\n",
+       "      <td>COMPLETED</td>\n",
+       "      <td>...</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>NaN</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>3927198</td>\n",
+       "      <td>3887451</td>\n",
+       "      <td>30.0</td>\n",
+       "      <td>batch</td>\n",
+       "      <td>3887451_30.batch</td>\n",
+       "      <td>batch</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>user</td>\n",
+       "      <td>COMPLETED</td>\n",
+       "      <td>...</td>\n",
+       "      <td>c0088</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1.222336e+10</td>\n",
+       "      <td>c0088</td>\n",
+       "      <td>0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>3927198</td>\n",
+       "      <td>3887451</td>\n",
+       "      <td>30.0</td>\n",
+       "      <td>extern</td>\n",
+       "      <td>3887451_30.extern</td>\n",
+       "      <td>extern</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>user</td>\n",
+       "      <td>COMPLETED</td>\n",
+       "      <td>...</td>\n",
+       "      <td>c0088</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.000000e+00</td>\n",
+       "      <td>c0088</td>\n",
+       "      <td>0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>3927199</td>\n",
+       "      <td>3887451</td>\n",
+       "      <td>31.0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>3887451_31</td>\n",
+       "      <td>100kCrC20MPa</td>\n",
+       "      <td>user</td>\n",
+       "      <td>user</td>\n",
+       "      <td>user</td>\n",
+       "      <td>COMPLETED</td>\n",
+       "      <td>...</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>NaN</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>5 rows × 63 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "     JobID  ArrayJobID  ArrayTaskID JobStep            JobIDSlurm  \\\n",
+       "0  3319116     3319116          NaN    None  3319116_[43-45,47%5]   \n",
+       "1  3927198     3887451         30.0    None            3887451_30   \n",
+       "2  3927198     3887451         30.0   batch      3887451_30.batch   \n",
+       "3  3927198     3887451         30.0  extern     3887451_30.extern   \n",
+       "4  3927199     3887451         31.0    None            3887451_31   \n",
+       "\n",
+       "        JobName      User     Group   Account      State  ...  \\\n",
+       "0      1mUD1MPa  user  user  user    PENDING  ...   \n",
+       "1  100kCrC20MPa  user  user  user  COMPLETED  ...   \n",
+       "2         batch                      user  COMPLETED  ...   \n",
+       "3        extern                      user  COMPLETED  ...   \n",
+       "4  100kCrC20MPa  user  user  user  COMPLETED  ...   \n",
+       "\n",
+       "   MaxDiskReadNode  MaxDiskReadTask  MaxDiskWrite  MaxDiskWriteNode  \\\n",
+       "0                                             NaN                     \n",
+       "1                                             NaN                     \n",
+       "2            c0088                0  1.222336e+10             c0088   \n",
+       "3            c0088                0  0.000000e+00             c0088   \n",
+       "4                                             NaN                     \n",
+       "\n",
+       "   MaxDiskWriteTask  ReqGPUS Comment GPUMem  GPUEff  NGPU  \n",
+       "0                        NaN    None   None    None  None  \n",
+       "1                        NaN    None   None    None  None  \n",
+       "2                 0      NaN    None   None    None  None  \n",
+       "3                 0      NaN    None   None    None  None  \n",
+       "4                        NaN    None   None    None  None  \n",
+       "\n",
+       "[5 rows x 63 columns]"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df1.head(5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python [conda env:.conda-slurm-ds]",
+   "language": "python",
+   "name": "conda-env-.conda-slurm-ds-py"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}