diff --git a/slurm-2sql.ipynb b/slurm-2sql.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..f2608ac87140174fb649f902e4175ef7b5bfc8eb --- /dev/null +++ b/slurm-2sql.ipynb @@ -0,0 +1,290 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "import sqlite3\n", + "import slurm2sql\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "db = sqlite3.connect('test.db')\n", + "slurm2sql.slurm2sql(db, ['-S', '2020-03-18', '-a'])" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "# For example, you can then convert to a dataframe:\n", + "df1 = pd.read_sql('SELECT * FROM slurm', db)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "<div>\n", + "<style scoped>\n", + " .dataframe tbody tr th:only-of-type {\n", + " vertical-align: middle;\n", + " }\n", + "\n", + " .dataframe tbody tr th {\n", + " vertical-align: top;\n", + " }\n", + "\n", + " .dataframe thead th {\n", + " text-align: right;\n", + " }\n", + "</style>\n", + "<table border=\"1\" class=\"dataframe\">\n", + " <thead>\n", + " <tr style=\"text-align: right;\">\n", + " <th></th>\n", + " <th>JobID</th>\n", + " <th>ArrayJobID</th>\n", + " <th>ArrayTaskID</th>\n", + " <th>JobStep</th>\n", + " <th>JobIDSlurm</th>\n", + " <th>JobName</th>\n", + " <th>User</th>\n", + " <th>Group</th>\n", + " <th>Account</th>\n", + " <th>State</th>\n", + " <th>...</th>\n", + " <th>MaxDiskReadNode</th>\n", + " <th>MaxDiskReadTask</th>\n", + " <th>MaxDiskWrite</th>\n", + " <th>MaxDiskWriteNode</th>\n", + " <th>MaxDiskWriteTask</th>\n", + " <th>ReqGPUS</th>\n", + " <th>Comment</th>\n", + " <th>GPUMem</th>\n", + " <th>GPUEff</th>\n", + " <th>NGPU</th>\n", + " </tr>\n", + " </thead>\n", + " <tbody>\n", + " <tr>\n", + " <th>0</th>\n", + " <td>3319116</td>\n", + " <td>3319116</td>\n", + " <td>NaN</td>\n", + " <td>None</td>\n", + " <td>3319116_[43-45,47%5]</td>\n", + " <td>1mUD1MPa</td>\n", + " <td>user</td>\n", + " <td>user</td>\n", + " <td>user</td>\n", + " <td>PENDING</td>\n", + " <td>...</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>NaN</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>NaN</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " </tr>\n", + " <tr>\n", + " <th>1</th>\n", + " <td>3927198</td>\n", + " <td>3887451</td>\n", + " <td>30.0</td>\n", + " <td>None</td>\n", + " <td>3887451_30</td>\n", + " <td>100kCrC20MPa</td>\n", + " <td>user</td>\n", + " <td>user</td>\n", + " <td>user</td>\n", + " <td>COMPLETED</td>\n", + " <td>...</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>NaN</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>NaN</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " </tr>\n", + " <tr>\n", + " <th>2</th>\n", + " <td>3927198</td>\n", + " <td>3887451</td>\n", + " <td>30.0</td>\n", + " <td>batch</td>\n", + " <td>3887451_30.batch</td>\n", + " <td>batch</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>user</td>\n", + " <td>COMPLETED</td>\n", + " <td>...</td>\n", + " <td>c0088</td>\n", + " <td>0</td>\n", + " <td>1.222336e+10</td>\n", + " <td>c0088</td>\n", + " <td>0</td>\n", + " <td>NaN</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " </tr>\n", + " <tr>\n", + " <th>3</th>\n", + " <td>3927198</td>\n", + " <td>3887451</td>\n", + " <td>30.0</td>\n", + " <td>extern</td>\n", + " <td>3887451_30.extern</td>\n", + " <td>extern</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>user</td>\n", + " <td>COMPLETED</td>\n", + " <td>...</td>\n", + " <td>c0088</td>\n", + " <td>0</td>\n", + " <td>0.000000e+00</td>\n", + " <td>c0088</td>\n", + " <td>0</td>\n", + " <td>NaN</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " </tr>\n", + " <tr>\n", + " <th>4</th>\n", + " <td>3927199</td>\n", + " <td>3887451</td>\n", + " <td>31.0</td>\n", + " <td>None</td>\n", + " <td>3887451_31</td>\n", + " <td>100kCrC20MPa</td>\n", + " <td>user</td>\n", + " <td>user</td>\n", + " <td>user</td>\n", + " <td>COMPLETED</td>\n", + " <td>...</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>NaN</td>\n", + " <td></td>\n", + " <td></td>\n", + " <td>NaN</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " <td>None</td>\n", + " </tr>\n", + " </tbody>\n", + "</table>\n", + "<p>5 rows × 63 columns</p>\n", + "</div>" + ], + "text/plain": [ + " JobID ArrayJobID ArrayTaskID JobStep JobIDSlurm \\\n", + "0 3319116 3319116 NaN None 3319116_[43-45,47%5] \n", + "1 3927198 3887451 30.0 None 3887451_30 \n", + "2 3927198 3887451 30.0 batch 3887451_30.batch \n", + "3 3927198 3887451 30.0 extern 3887451_30.extern \n", + "4 3927199 3887451 31.0 None 3887451_31 \n", + "\n", + " JobName User Group Account State ... \\\n", + "0 1mUD1MPa user user user PENDING ... \n", + "1 100kCrC20MPa user user user COMPLETED ... \n", + "2 batch user COMPLETED ... \n", + "3 extern user COMPLETED ... \n", + "4 100kCrC20MPa user user user COMPLETED ... \n", + "\n", + " MaxDiskReadNode MaxDiskReadTask MaxDiskWrite MaxDiskWriteNode \\\n", + "0 NaN \n", + "1 NaN \n", + "2 c0088 0 1.222336e+10 c0088 \n", + "3 c0088 0 0.000000e+00 c0088 \n", + "4 NaN \n", + "\n", + " MaxDiskWriteTask ReqGPUS Comment GPUMem GPUEff NGPU \n", + "0 NaN None None None None \n", + "1 NaN None None None None \n", + "2 0 NaN None None None None \n", + "3 0 NaN None None None None \n", + "4 NaN None None None None \n", + "\n", + "[5 rows x 63 columns]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df1.head(5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:.conda-slurm-ds]", + "language": "python", + "name": "conda-env-.conda-slurm-ds-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}