From e74eee3ce471de3739126f9f65363d004515fabd Mon Sep 17 00:00:00 2001
From: Rishi Sharma <rishi.sharma@epfl.ch>
Date: Thu, 10 Feb 2022 17:33:29 +0100
Subject: [PATCH] Add model ratio plotting
---
eval/plot_model.py | 92 +++++++++++++++++++
.../training/ChangeAccumulator.py | 2 +-
2 files changed, 93 insertions(+), 1 deletion(-)
create mode 100644 eval/plot_model.py
diff --git a/eval/plot_model.py b/eval/plot_model.py
new file mode 100644
index 0000000..1d45040
--- /dev/null
+++ b/eval/plot_model.py
@@ -0,0 +1,92 @@
+import json
+import os
+import sys
+from pathlib import Path
+
+import numpy as np
+from matplotlib import pyplot as plt
+from pyexpat import model
+
+
+def plot(x, y, label, *args):
+ plt.plot(x, y, *args, label=label)
+ plt.legend()
+
+
+def reject_outliers(data, m=2.0):
+ d = np.abs(data - np.median(data))
+ mdev = np.median(d)
+ s = d / (mdev if mdev else 1.0)
+ return data[s < m]
+
+
+def plot_model(path, title):
+ model_path = os.path.join(path, "plots")
+ Path(model_path).mkdir(parents=True, exist_ok=True)
+ files = [f for f in os.listdir(path) if f.endswith("json")]
+ for file in files:
+ filepath = os.path.join(path, file)
+ with open(filepath, "r") as inf:
+ model_vec = json.load(inf)
+ del model_vec["order"]
+ del model_vec["shapes"]
+ model_vec = np.array(model_vec[list(model_vec.keys())[0]])
+ num_elements = model_vec.shape[0]
+ x_axis = np.arange(1, num_elements + 1)
+ plt.clf()
+ plt.title(title)
+ plot(x_axis, model_vec, "unsorted", ".")
+ model_vec = np.sort(model_vec)
+ plot(x_axis, model_vec, "sorted")
+ plt.savefig(os.path.join(model_path, file[0:-5]))
+
+
+def plot_ratio(path_change, path_val, title):
+ model_path = os.path.join(path_change, "plots_ratio")
+ Path(model_path).mkdir(parents=True, exist_ok=True)
+ files_change = [f for f in os.listdir(path_change) if f.endswith("json")]
+ files_val = [f for f in os.listdir(path_val) if f.endswith("json")]
+ for i, file in enumerate(files_change):
+ print("Processed ", file)
+ filepath_change = os.path.join(path_change, file)
+ filepath_val = os.path.join(path_val, files_val[i])
+ with open(filepath_change, "r") as inf:
+ model_change = json.load(inf)
+ del model_change["order"]
+ del model_change["shapes"]
+ model_change = np.array(model_change[list(model_change.keys())[0]])
+ with open(filepath_val, "r") as inf:
+ model_val = json.load(inf)
+ del model_val["order"]
+ del model_val["shapes"]
+ model_val = np.array(model_val[list(model_val.keys())[0]])
+ num_elements = model_val.shape[0]
+ x_axis = np.arange(1, num_elements + 1)
+ plt.clf()
+ plt.title(title)
+ model_vec = np.divide(
+ model_change,
+ model_val,
+ out=np.zeros_like(model_change),
+ where=model_val != 0.0,
+ )
+ model_vec = reject_outliers(model_vec)
+ num_elements = model_vec.shape[0]
+ x_axis = np.arange(1, num_elements + 1)
+ plot(x_axis, model_vec, "unsorted", ".")
+ model_vec = np.sort(model_vec)
+ plot(x_axis, model_vec, "sorted")
+ plt.savefig(os.path.join(model_path, file[0:-5]))
+
+
+if __name__ == "__main__":
+ assert len(sys.argv) == 3
+ plot_model(
+ os.path.join(sys.argv[1], "model_change", sys.argv[2]), "Change in Weights"
+ )
+ plot_model(os.path.join(sys.argv[1], "model_val", sys.argv[2]), "Model Parameters")
+ plot_ratio(
+ os.path.join(sys.argv[1], "model_change", sys.argv[2]),
+ os.path.join(sys.argv[1], "model_val", sys.argv[2]),
+ "Ratio",
+ )
diff --git a/src/decentralizepy/training/ChangeAccumulator.py b/src/decentralizepy/training/ChangeAccumulator.py
index a463a5b..257b4ac 100644
--- a/src/decentralizepy/training/ChangeAccumulator.py
+++ b/src/decentralizepy/training/ChangeAccumulator.py
@@ -105,7 +105,7 @@ class ChangeAccumulator(Training):
shapes[k] = list(v1.shape)
output_dict["shapes"] = shapes
- output_dict[self.communication_round] = v.tolist()
+ output_dict["tensor"] = v.tolist()
with open(
os.path.join(
--
GitLab