diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..3319e31e869876559896347734d7946c4cbb113b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,85 @@ +astroid==2.3.3 +astropy==4.0 +attr==0.3.1 +attrs==19.3.0 +backcall==0.1.0 +bleach==3.1.3 +certifi==2019.11.28 +chardet==3.0.4 +confuse==1.0.0 +cycler==0.10.0 +decorator==4.4.2 +defusedxml==0.6.0 +entrypoints==0.3 +htmlmin==0.1.12 +idna==2.8 +ipykernel==5.1.4 +ipython==7.13.0 +ipython-genutils==0.2.0 +ipywidgets==7.5.1 +isort==4.3.21 +jedi==0.16.0 +Jinja2==2.11.1 +joblib==0.14.1 +jsonschema==3.2.0 +jupyter-client==6.0.0 +jupyter-core==4.6.1 +kaggle==1.5.6 +kiwisolver==1.1.0 +lazy-object-proxy==1.4.3 +llvmlite==0.31.0 +MarkupSafe==1.1.1 +matplotlib==3.2.0 +mccabe==0.6.1 +missingno==0.4.2 +mistune==0.8.4 +more-itertools==8.2.0 +nbconvert==5.6.1 +nbformat==5.0.4 +networkx==2.4 +notebook==6.0.3 +numba==0.48.0 +numpy==1.18.2 +packaging==20.3 +pandas==0.25.3 +pandas-profiling==2.5.0 +pandocfilters==1.4.2 +parso==0.6.2 +pexpect==4.8.0 +phik==0.9.9 +pickleshare==0.7.5 +pluggy==0.13.1 +prometheus-client==0.7.1 +prompt-toolkit==3.0.3 +ptyprocess==0.6.0 +py==1.8.1 +Pygments==2.6.1 +pylint==2.4.4 +pyparsing==2.4.6 +pyrsistent==0.15.7 +pytest==5.4.1 +pytest-pylint==0.15.1 +python-dateutil==2.8.1 +python-slugify==4.0.0 +pytz==2019.3 +PyYAML==5.3 +pyzmq==18.1.1 +requests==2.22.0 +scipy==1.4.1 +seaborn==0.10.0 +Send2Trash==1.5.0 +six==1.14.0 +slurm2sql==0.9.0 +tangled-up-in-unicode==0.0.3 +terminado==0.8.3 +testpath==0.4.4 +text-unidecode==1.3 +tornado==6.0.4 +tqdm==4.42.0 +traitlets==4.3.3 +urllib3==1.25.8 +visions==0.2.2 +wcwidth==0.1.8 +webencodings==0.5.1 +widgetsnbextension==3.5.1 +wrapt==1.11.2 diff --git a/slurm-2sql.ipynb b/slurm-2sql.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..f2608ac87140174fb649f902e4175ef7b5bfc8eb --- /dev/null +++ b/slurm-2sql.ipynb @@ -0,0 +1,290 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "import sqlite3\n", + "import slurm2sql\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db = sqlite3.connect('test.db')\n", + "slurm2sql.slurm2sql(db, ['-S', '2020-03-18', '-a'])" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "# For example, you can then convert to a dataframe:\n", + "df1 = pd.read_sql('SELECT * FROM slurm', db)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>JobID</th>\n", + " <th>ArrayJobID</th>\n", + " <th>ArrayTaskID</th>\n", + " <th>JobStep</th>\n", + " <th>JobIDSlurm</th>\n", + " <th>JobName</th>\n", + " <th>User</th>\n", + " <th>Group</th>\n", + " <th>Account</th>\n", + " <th>State</th>\n", + " <th>...</th>\n", + " <th>MaxDiskReadNode</th>\n", + " <th>MaxDiskReadTask</th>\n", + " <th>MaxDiskWrite</th>\n", + " <th>MaxDiskWriteNode</th>\n", + " <th>MaxDiskWriteTask</th>\n", + " <th>ReqGPUS</th>\n", + " <th>Comment</th>\n", + " <th>GPUMem</th>\n", + " <th>GPUEff</th>\n", + " <th>NGPU</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>0</th>\n", + " <td>3319116</td>\n", + " <td>3319116</td>\n", + " <td>NaN</td>\n", + " <td>None</td>\n", + " <td>3319116_[43-45,47%5]</td>\n", + " <td>1mUD1MPa</td>\n", + " <td>user</td>\n", + " <td>user</td>\n", + " <td>user</td>\n", + " <td>PENDING</td>\n", + " <td>...</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>NaN</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>NaN</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " </tr>\n", + " <tr>\n", + " <th>1</th>\n", + " <td>3927198</td>\n", + " <td>3887451</td>\n", + " <td>30.0</td>\n", + " <td>None</td>\n", + " <td>3887451_30</td>\n", + " <td>100kCrC20MPa</td>\n", + " <td>user</td>\n", + " <td>user</td>\n", + " <td>user</td>\n", + " <td>COMPLETED</td>\n", + " <td>...</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>NaN</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>NaN</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " </tr>\n", + " <tr>\n", + " <th>2</th>\n", + " <td>3927198</td>\n", + " <td>3887451</td>\n", + " <td>30.0</td>\n", + " <td>batch</td>\n", + " <td>3887451_30.batch</td>\n", + " <td>batch</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>user</td>\n", + " <td>COMPLETED</td>\n", + " <td>...</td>\n", + " <td>c0088</td>\n", + " <td>0</td>\n", + " <td>1.222336e+10</td>\n", + " <td>c0088</td>\n", + " <td>0</td>\n", + " <td>NaN</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " </tr>\n", + " <tr>\n", + " <th>3</th>\n", + " <td>3927198</td>\n", + " <td>3887451</td>\n", + " <td>30.0</td>\n", + " <td>extern</td>\n", + " <td>3887451_30.extern</td>\n", + " <td>extern</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>user</td>\n", + " <td>COMPLETED</td>\n", + " <td>...</td>\n", + " <td>c0088</td>\n", + " <td>0</td>\n", + " <td>0.000000e+00</td>\n", + " <td>c0088</td>\n", + " <td>0</td>\n", + " <td>NaN</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " </tr>\n", + " <tr>\n", + " <th>4</th>\n", + " <td>3927199</td>\n", + " <td>3887451</td>\n", + " <td>31.0</td>\n", + " <td>None</td>\n", + " <td>3887451_31</td>\n", + " <td>100kCrC20MPa</td>\n", + " <td>user</td>\n", + " <td>user</td>\n", + " <td>user</td>\n", + " <td>COMPLETED</td>\n", + " <td>...</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>NaN</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>NaN</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "<p>5 rows × 63 columns</p>\n", + "</div>" + ], + "text/plain": [ + " JobID ArrayJobID ArrayTaskID JobStep JobIDSlurm \\\n", + "0 3319116 3319116 NaN None 3319116_[43-45,47%5] \n", + "1 3927198 3887451 30.0 None 3887451_30 \n", + "2 3927198 3887451 30.0 batch 3887451_30.batch \n", + "3 3927198 3887451 30.0 extern 3887451_30.extern \n", + "4 3927199 3887451 31.0 None 3887451_31 \n", + "\n", + " JobName User Group Account State ... \\\n", + "0 1mUD1MPa user user user PENDING ... \n", + "1 100kCrC20MPa user user user COMPLETED ... \n", + "2 batch user COMPLETED ... \n", + "3 extern user COMPLETED ... \n", + "4 100kCrC20MPa user user user COMPLETED ... \n", + "\n", + " MaxDiskReadNode MaxDiskReadTask MaxDiskWrite MaxDiskWriteNode \\\n", + "0 NaN \n", + "1 NaN \n", + "2 c0088 0 1.222336e+10 c0088 \n", + "3 c0088 0 0.000000e+00 c0088 \n", + "4 NaN \n", + "\n", + " MaxDiskWriteTask ReqGPUS Comment GPUMem GPUEff NGPU \n", + "0 NaN None None None None \n", + "1 NaN None None None None \n", + "2 0 NaN None None None None \n", + "3 0 NaN None None None None \n", + "4 NaN None None None None \n", + "\n", + "[5 rows x 63 columns]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df1.head(5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:.conda-slurm-ds]", + "language": "python", + "name": "conda-env-.conda-slurm-ds-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}