From 32cb5a451699f3d22a6b54fb743364b827f17459 Mon Sep 17 00:00:00 2001
From: John-Paul Robinson <jpr@uab.edu>
Date: Thu, 22 Apr 2021 11:09:33 -0500
Subject: [PATCH] Batch update for multiple changes
Turn off cert verify on query.
Change default date ranges and inspected nodes (still needs to
be more data driven).
Add better logic for checking for saved data.
Add save data logic at end of notebook.
---
power-stats.ipynb | 199 +++++++++++++++++++++++++++++++++++++++++-----
1 file changed, 180 insertions(+), 19 deletions(-)
diff --git a/power-stats.ipynb b/power-stats.ipynb
index 24a2d96..60efb68 100644
--- a/power-stats.ipynb
+++ b/power-stats.ipynb
@@ -86,8 +86,18 @@
"metadata": {},
"outputs": [],
"source": [
- "startdate = '2020/01/01 00:00:00'\n",
- "enddate = '2021/02/21 00:00:00'"
+ "startdate = '2021/01/01 00:00:00'\n",
+ "enddate = '2021/04/8 00:00:00'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "displaystart = '2021-02-01'\n",
+ "displaystop = '2021-04-08'"
]
},
{
@@ -109,7 +119,11 @@
"metadata": {},
"outputs": [],
"source": [
- "response = requests.get('https://master:8081/rest/v1/monitoring/dump', params=params, cert=cert, verify=False)"
+ "if os.path.exists(\"ipower_data.csv\"):\n",
+ " df = pd.read_csv(\"power_data.csv\")\n",
+ "else:\n",
+ " response = requests.get('https://master:8081/rest/v1/monitoring/dump', params=params, cert=cert, verify=False)\n",
+ " df = pd.DataFrame(response.json()[\"data\"])"
]
},
{
@@ -119,15 +133,6 @@
"Simply read the json response into a dataframe for futher parsing."
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "df = pd.DataFrame(response.json()[\"data\"])"
- ]
- },
{
"cell_type": "markdown",
"metadata": {},
@@ -148,6 +153,15 @@
"#df = df.loc[df['raw'] < 10000]"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -192,7 +206,7 @@
"m6_hourly_pwr=pd.DataFrame(np.zeros((1,len(hourly_idx))).T, index=hourly_idx, columns=['sum'])\n",
"\n",
"for num, entity in enumerate(sorted(df.entity.unique())):\n",
- " if entity not in ['c0108', 'c0009']:\n",
+ " if entity not in ['c0009']:\n",
" node_pwr=df[df.entity==entity].set_index(\"datetime\")\n",
" node_pwr=node_pwr[['raw']].resample('H').mean()\n",
" node_pwr=node_pwr[startdate:enddate].fillna(method=\"ffill\")\n",
@@ -204,11 +218,20 @@
" m6_hourly_pwr[entity]= node_pwr[startdate:enddate]\n"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "m6_hourly_pwr"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "## Plot Per-node Hourly\n",
+ "## Plot Per-node Hourly for Row 5 Rack 1\n",
"\n",
"This is just to see the data for each node in one plot and get a feel for how the nodes behave relative to each other. Plot nodes in individual subplotes to decern individual behavior of specific nodes. It does give a sense of how the total power adds up. \n",
"\n",
@@ -228,7 +251,7 @@
"num_nodes=36\n",
"fig, axes = plt.subplots(num_nodes,1, sharex=True, figsize=(20,30))\n",
"for i in range(num_nodes):\n",
- " m6_hourly_pwr['2020-02-01':'2021-02-21'].iloc[:,i+1:i+2].plot(ax=axes[i], legend=True)\n",
+ " m6_hourly_pwr[displaystart:displaystop].iloc[:,i+1:i+2].plot(ax=axes[i], legend=True)\n",
" axes[i].legend(loc='lower left')"
]
},
@@ -245,7 +268,61 @@
"metadata": {},
"outputs": [],
"source": [
- "m6_hourly_pwr['2020-02-01':'2021-02-21'].iloc[:,1:2].plot()"
+ "select_node=\"c0022\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df[df.entity==select_node].set_index(\"datetime\")[\"2020-09-01\":\"2020-10-04\"]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "m6_hourly_pwr[displaystart:displaystop].iloc[:,3:4][\"2020-10-03\":\"2020-10-04\"]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "m6_hourly_pwr[displaystart:displaystop].iloc[:,3:4][\"2020-09-28\":\"2020-10-14\"].plot()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "m6_hourly_pwr[displaystart:displaystop].iloc[:,3:4].plot()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df[df[\"entity\"]==\"c0001\"]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df[df[\"entity\"]==\"c0001\"][\"datetime\"].max()"
]
},
{
@@ -263,7 +340,7 @@
"metadata": {},
"outputs": [],
"source": [
- "nan_mask = m6_hourly_pwr['2021-02-01':'2021-02-02'].isna()"
+ "nan_mask = m6_hourly_pwr[\"2021-03-22\":\"2021-03-23\"].isna()"
]
},
{
@@ -275,6 +352,15 @@
"power_missing = nan_mask[nan_mask].apply(lambda row: row[row == True].index, axis=1)[1]"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(*power_missing,sep=\", \")"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -288,6 +374,54 @@
" axes[i].legend(loc='lower left')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "node"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df[df[\"entity\"]==node][\"datetime\"].max()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "lastreport = pd.DataFrame(columns=('node', 'datetime'))\n",
+ "\n",
+ "for i, node in enumerate(power_missing):\n",
+ " lastreport.loc[i] = [node, df[df[\"entity\"]==node][\"datetime\"].max()]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "lastreport.sort_values(by=\"datetime\") #[\"datetime\"].sort()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(\"{}:\\t{}\".format(node, df[df[\"entity\"]==node][\"datetime\"].max()))"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -309,7 +443,7 @@
"for i, node in enumerate(m6_hourly_pwr.iloc[:,1:].columns):\n",
" if (i == num_nodes):\n",
" break\n",
- " m6_hourly_pwr[node]['2020-02-01':'2021-02-21'].plot(ax=axes[i], legend=True)\n",
+ " m6_hourly_pwr[node][displaystart:displaystop].plot(ax=axes[i], legend=True)\n",
" axes[i].legend(loc='lower left')\n"
]
},
@@ -331,7 +465,7 @@
"metadata": {},
"outputs": [],
"source": [
- "kW = m6_hourly_pwr['2020-02-01':'2021-02-21'].sum(axis=1)/1000"
+ "kW = m6_hourly_pwr[displaystart:displaystop].sum(axis=1)/1000"
]
},
{
@@ -388,6 +522,33 @@
"ax.set_ylabel('Power (kW)')\n",
"ax.set_title('Cheaha Trends in Electricity Consumption');"
]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Save Hourly Power to Dataframe\n",
+ "\n",
+ "This makes it easy to use the data in other analysis and learning efforts."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "m6_hourly_pwr.to_pickle(\"m6_hourly_pwr.gz\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df.to_pickle(\"power_stats_raw_df.gz\")"
+ ]
}
],
"metadata": {
--
GitLab