Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • sacs/decentralizepy
  • mvujas/decentralizepy
  • randl/decentralizepy
3 results
Show changes
Commits on Source (20)
Hide whitespace changes
Inline Side-by-side
Showing
with 758 additions and 306 deletions
.gitignore
View file @ d140bf44
...@@ -8,4 +8,6 @@ ...@@ -8,4 +8,6 @@
**/leaf/ **/leaf/
**.egg-info **.egg-info
2021** 2021**
2022** 2022**
\ No newline at end of file **/massif.out*
*swp
README.rst
View file @ d140bf44
...@@ -2,6 +2,9 @@ ...@@ -2,6 +2,9 @@
decentralizepy decentralizepy
============== ==============
decentralizepy is a framework for running distributed applications (particularly ML) on top of arbitrary topologies (decentralized, federated, parameter server).
It was primarily conceived for assessing scientific ideas on several aspects of distributed learning (communication efficiency, privacy, data heterogeneity etc.).
------------------------- -------------------------
Setting up decentralizepy Setting up decentralizepy
------------------------- -------------------------
...@@ -23,10 +26,14 @@ Setting up decentralizepy ...@@ -23,10 +26,14 @@ Setting up decentralizepy
pip install --upgrade pip pip install --upgrade pip
* On Mac M1, installing ``pyzmq`` fails with `pip`. Use `conda <https://conda.io>`_. * On Mac M1, installing ``pyzmq`` fails with `pip`. Use `conda <https://conda.io>`_.
* Install decentralizepy for development. :: * Install decentralizepy for development. (zsh) ::
pip3 install --editable .\[dev\] pip3 install --editable .\[dev\]
* Install decentralizepy for development. (bash) ::
pip3 install --editable .[dev]
---------------- ----------------
Running the code Running the code
---------------- ----------------
......
eval/main.ipynb
View file @ d140bf44
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
from datasets.Femnist import Femnist from datasets.Femnist import Femnist
from graphs import SmallWorld from graphs import SmallWorld
from collections import defaultdict from collections import defaultdict
import os import os
import json import json
import numpy as np import numpy as np
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
a = FEMNIST a = FEMNIST
a a
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
b = SmallWorld(6, 2, 2, 1) b = SmallWorld(6, 2, 2, 1)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
b.adj_list b.adj_list
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
for i in range(12): for i in range(12):
print(b.neighbors(i)) print(b.neighbors(i))
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
clients = [] clients = []
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
num_samples = [] num_samples = []
data = defaultdict(lambda : None) data = defaultdict(lambda : None)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
datadir = "./leaf/data/femnist/data/train" datadir = "./leaf/data/femnist/data/train"
files = os.listdir(datadir) files = os.listdir(datadir)
total_users=0 total_users=0
users = set() users = set()
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
files = os.listdir(datadir)[0:1] files = os.listdir(datadir)[0:1]
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
for f in files: for f in files:
file_path = os.path.join(datadir, f) file_path = os.path.join(datadir, f)
print(file_path) print(file_path)
with open(file_path, 'r') as inf: with open(file_path, 'r') as inf:
client_data = json.load(inf) client_data = json.load(inf)
current_users = len(client_data['users']) current_users = len(client_data['users'])
print("Current_Users: ", current_users) print("Current_Users: ", current_users)
total_users += current_users total_users += current_users
users.update(client_data['users']) users.update(client_data['users'])
print("total_users: ", total_users) print("total_users: ", total_users)
print("total_users: ", len(users)) print("total_users: ", len(users))
print(client_data['user_data'].keys()) print(client_data['user_data'].keys())
print(np.array(client_data['user_data']['f3408_47']['x']).shape) print(np.array(client_data['user_data']['f3408_47']['x']).shape)
print(np.array(client_data['user_data']['f3408_47']['y']).shape) print(np.array(client_data['user_data']['f3408_47']['y']).shape)
print(np.array(client_data['user_data']['f3327_11']['x']).shape) print(np.array(client_data['user_data']['f3327_11']['x']).shape)
print(np.array(client_data['user_data']['f3327_11']['y']).shape) print(np.array(client_data['user_data']['f3327_11']['y']).shape)
print(np.unique(np.array(client_data['user_data']['f3327_11']['y']))) print(np.unique(np.array(client_data['user_data']['f3327_11']['y'])))
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
file = 'run.py' file = 'run.py'
with open(file, 'r') as inf: with open(file, 'r') as inf:
print(inf.readline().strip()) print(inf.readline().strip())
print(inf.readlines()) print(inf.readlines())
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
def f(l): def f(l):
l[2] = 'c' l[2] = 'c'
a = ['a', 'a', 'a'] a = ['a', 'a', 'a']
print(a) print(a)
f(a) f(a)
print(a) print(a)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
l = ['a', 'b', 'c'] l = ['a', 'b', 'c']
print(l[:-1]) print(l[:-1])
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
from localconfig import LocalConfig from localconfig import LocalConfig
def read_ini(file_path): def read_ini(file_path):
config = LocalConfig(file_path) config = LocalConfig(file_path)
for section in config: for section in config:
print("Section: ", section) print("Section: ", section)
for key, value in config.items(section): for key, value in config.items(section):
print((key, value)) print((key, value))
print(dict(config.items('DATASET'))) print(dict(config.items('DATASET')))
return config return config
config = read_ini("config.ini") config = read_ini("config.ini")
for section in config: for section in config:
print(section) print(section)
#d = dict(config.sections()) #d = dict(config.sections())
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
def func(a = 1, b = 2, c = 3): def func(a = 1, b = 2, c = 3):
print(a + b + c) print(a + b + c)
l = [3, 5, 7] l = [3, 5, 7]
func(*l) func(*l)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
from torch import multiprocessing as mp from torch import multiprocessing as mp
mp.spawn(fn = func, nprocs = 2, args = [], kwargs = {'a': 4, 'b': 5, 'c': 6}) mp.spawn(fn = func, nprocs = 2, args = [], kwargs = {'a': 4, 'b': 5, 'c': 6})
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
l = '[0.4, 0.2, 0.3, 0.1]' l = '[0.4, 0.2, 0.3, 0.1]'
type(eval(l)) type(eval(l))
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
from decentralizepy.datasets.Femnist import Femnist from decentralizepy.datasets.Femnist import Femnist
f1 = Femnist(0, 1, 'leaf/data/femnist/data/train') f1 = Femnist(0, 1, 'leaf/data/femnist/data/train')
ts = f1.get_trainset(1) ts = f1.get_trainset(1)
for data, target in ts: for data, target in ts:
print(data) print(data)
break break
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
from decentralizepy.datasets.Femnist import Femnist from decentralizepy.datasets.Femnist import Femnist
from decentralizepy.graphs.SmallWorld import SmallWorld from decentralizepy.graphs.SmallWorld import SmallWorld
from decentralizepy.mappings.Linear import Linear from decentralizepy.mappings.Linear import Linear
f = Femnist(2, 'leaf/data/femnist/data/train', sizes=[0.6, 0.4]) f = Femnist(2, 'leaf/data/femnist/data/train', sizes=[0.6, 0.4])
g = SmallWorld(4, 1, 0.5) g = SmallWorld(4, 1, 0.5)
l = Linear(2, 2) l = Linear(2, 2)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
from decentralizepy.node.Node import Node from decentralizepy.node.Node import Node
from torch import multiprocessing as mp from torch import multiprocessing as mp
import logging import logging
n1 = Node(0, l, g, f, "./results", logging.DEBUG) n1 = Node(0, l, g, f, "./results", logging.DEBUG)
n2 = Node(1, l, g, f, "./results", logging.DEBUG) n2 = Node(1, l, g, f, "./results", logging.DEBUG)
# mp.spawn(fn = Node, nprocs = 2, args=[l,g,f]) # mp.spawn(fn = Node, nprocs = 2, args=[l,g,f])
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
from testing import f from testing import f
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
from torch import multiprocessing as mp from torch import multiprocessing as mp
import torch import torch
from decentralizepy.datasets.Celeba import CNN from decentralizepy.datasets.Celeba import CNN
import numpy as np import numpy as np
m1 = CNN() m1 = CNN()
o1 = torch.optim.SGD(m1.parameters(), 0.6) o1 = torch.optim.SGD(m1.parameters(), 0.6)
print(m1) print(m1)
#mp.spawn(fn = f, nprocs = 2, args=[m1, o1]) #mp.spawn(fn = f, nprocs = 2, args=[m1, o1])
``` ```
%% Output %% Output
CNN( CNN(
(conv1): Conv2d(3, 32, kernel_size=(3, 3), stride=(1, 1), padding=same) (conv1): Conv2d(3, 32, kernel_size=(3, 3), stride=(1, 1), padding=same)
(pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False) (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
(conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=same) (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=same)
(conv3): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=same) (conv3): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=same)
(conv4): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=same) (conv4): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=same)
(fc1): Linear(in_features=800, out_features=2, bias=True) (fc1): Linear(in_features=800, out_features=2, bias=True)
) )
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
m1.parameters() m1.parameters()
``` ```
%% Output %% Output
<generator object Module.parameters at 0x111ec1ba0> <generator object Module.parameters at 0x111ec1ba0>
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
m1.state_dict() m1.state_dict()
``` ```
%% Output %% Output
OrderedDict([('conv1.weight', OrderedDict([('conv1.weight',
tensor([[[[-0.1261, 0.1833, -0.1406], tensor([[[[-0.1261, 0.1833, -0.1406],
[ 0.1324, -0.0685, 0.0938], [ 0.1324, -0.0685, 0.0938],
[ 0.0432, 0.1814, -0.0541]], [ 0.0432, 0.1814, -0.0541]],
[[-0.1776, -0.1839, -0.0111], [[-0.1776, -0.1839, -0.0111],
[ 0.0888, 0.0888, -0.1344], [ 0.0888, 0.0888, -0.1344],
[-0.1838, 0.1737, 0.1584]], [-0.1838, 0.1737, 0.1584]],
[[ 0.0417, 0.1064, -0.0156], [[ 0.0417, 0.1064, -0.0156],
[ 0.0667, 0.0856, -0.1746], [ 0.0667, 0.0856, -0.1746],
[ 0.0412, 0.1620, 0.0125]]], [ 0.0412, 0.1620, 0.0125]]],
[[[-0.0530, -0.1273, -0.0797], [[[-0.0530, -0.1273, -0.0797],
[ 0.0422, 0.1135, 0.0475], [ 0.0422, 0.1135, 0.0475],
[-0.0244, 0.1691, -0.1383]], [-0.0244, 0.1691, -0.1383]],
[[ 0.0822, -0.1317, -0.1692], [[ 0.0822, -0.1317, -0.1692],
[ 0.1373, 0.1388, 0.0103], [ 0.1373, 0.1388, 0.0103],
[-0.0481, 0.1105, 0.0631]], [-0.0481, 0.1105, 0.0631]],
[[-0.0352, 0.1259, -0.0530], [[-0.0352, 0.1259, -0.0530],
[-0.1394, -0.0281, 0.1844], [-0.1394, -0.0281, 0.1844],
[ 0.0082, 0.1187, 0.0211]]], [ 0.0082, 0.1187, 0.0211]]],
[[[ 0.0987, 0.0788, -0.1126], [[[ 0.0987, 0.0788, -0.1126],
[ 0.1769, 0.0763, -0.1767], [ 0.1769, 0.0763, -0.1767],
[-0.0570, 0.1156, 0.1770]], [-0.0570, 0.1156, 0.1770]],
[[ 0.0643, -0.0024, -0.0625], [[ 0.0643, -0.0024, -0.0625],
[ 0.0819, 0.0140, -0.1882], [ 0.0819, 0.0140, -0.1882],
[ 0.1325, -0.0632, -0.0202]], [ 0.1325, -0.0632, -0.0202]],
[[ 0.0053, 0.1042, -0.0058], [[ 0.0053, 0.1042, -0.0058],
[-0.1082, -0.1753, 0.1762], [-0.1082, -0.1753, 0.1762],
[-0.0501, 0.1166, 0.0561]]], [-0.0501, 0.1166, 0.0561]]],
[[[ 0.0358, -0.0685, -0.1278], [[[ 0.0358, -0.0685, -0.1278],
[ 0.0029, -0.1107, 0.1169], [ 0.0029, -0.1107, 0.1169],
[-0.1408, 0.1293, 0.1142]], [-0.1408, 0.1293, 0.1142]],
[[-0.0814, 0.0470, 0.0188], [[-0.0814, 0.0470, 0.0188],
[ 0.1538, 0.0137, 0.1128], [ 0.1538, 0.0137, 0.1128],
[-0.1597, 0.1432, 0.1370]], [-0.1597, 0.1432, 0.1370]],
[[ 0.1425, 0.1769, -0.0037], [[ 0.1425, 0.1769, -0.0037],
[-0.1080, -0.0805, -0.0195], [-0.1080, -0.0805, -0.0195],
[-0.1335, -0.1666, 0.1399]]], [-0.1335, -0.1666, 0.1399]]],
[[[ 0.1117, 0.1918, -0.1666], [[[ 0.1117, 0.1918, -0.1666],
[-0.1392, 0.0086, 0.0172], [-0.1392, 0.0086, 0.0172],
[-0.0721, -0.1711, 0.0344]], [-0.0721, -0.1711, 0.0344]],
[[ 0.1820, -0.0537, -0.0974], [[ 0.1820, -0.0537, -0.0974],
[ 0.0366, -0.0710, 0.1273], [ 0.0366, -0.0710, 0.1273],
[ 0.1132, -0.1594, 0.0878]], [ 0.1132, -0.1594, 0.0878]],
[[-0.0874, -0.0401, 0.1827], [[-0.0874, -0.0401, 0.1827],
[-0.0301, 0.1205, -0.0396], [-0.0301, 0.1205, -0.0396],
[-0.1143, -0.1007, 0.1561]]], [-0.1143, -0.1007, 0.1561]]],
[[[ 0.1522, -0.0012, -0.1785], [[[ 0.1522, -0.0012, -0.1785],
[-0.1833, -0.1828, -0.1643], [-0.1833, -0.1828, -0.1643],
[-0.1765, -0.1757, -0.0608]], [-0.1765, -0.1757, -0.0608]],
[[-0.0684, 0.0521, 0.1137], [[-0.0684, 0.0521, 0.1137],
[-0.0028, 0.0616, 0.0758], [-0.0028, 0.0616, 0.0758],
[-0.1736, 0.0667, 0.1229]], [-0.1736, 0.0667, 0.1229]],
[[ 0.1298, -0.1848, -0.1570], [[ 0.1298, -0.1848, -0.1570],
[-0.1052, -0.1172, -0.1223], [-0.1052, -0.1172, -0.1223],
[-0.1389, -0.0095, -0.0410]]], [-0.1389, -0.0095, -0.0410]]],
[[[ 0.0213, -0.0975, 0.0964], [[[ 0.0213, -0.0975, 0.0964],
[ 0.0535, -0.0775, 0.0790], [ 0.0535, -0.0775, 0.0790],
[-0.1796, -0.1468, 0.1036]], [-0.1796, -0.1468, 0.1036]],
[[-0.0403, 0.0646, -0.0932], [[-0.0403, 0.0646, -0.0932],
[ 0.1779, -0.1616, 0.0644], [ 0.1779, -0.1616, 0.0644],
[-0.0508, -0.1158, -0.0592]], [-0.0508, -0.1158, -0.0592]],
[[-0.1644, -0.1327, 0.0817], [[-0.1644, -0.1327, 0.0817],
[ 0.0320, -0.0213, -0.0946], [ 0.0320, -0.0213, -0.0946],
[-0.1106, 0.1463, -0.1642]]], [-0.1106, 0.1463, -0.1642]]],
[[[-0.0985, -0.1160, -0.0915], [[[-0.0985, -0.1160, -0.0915],
[ 0.1857, 0.0806, 0.1761], [ 0.1857, 0.0806, 0.1761],
[-0.0817, 0.1095, 0.0896]], [-0.0817, 0.1095, 0.0896]],
[[-0.0660, -0.1680, 0.1833], [[-0.0660, -0.1680, 0.1833],
[ 0.0611, 0.0077, -0.0848], [ 0.0611, 0.0077, -0.0848],
[-0.1516, 0.1737, 0.0484]], [-0.1516, 0.1737, 0.0484]],
[[ 0.1434, -0.0732, -0.0904], [[ 0.1434, -0.0732, -0.0904],
[ 0.0962, 0.1783, 0.0192], [ 0.0962, 0.1783, 0.0192],
[ 0.0915, 0.0006, 0.0334]]], [ 0.0915, 0.0006, 0.0334]]],
[[[-0.0047, 0.1807, -0.1798], [[[-0.0047, 0.1807, -0.1798],
[-0.0164, 0.1119, -0.0805], [-0.0164, 0.1119, -0.0805],
[ 0.1855, -0.0681, -0.0187]], [ 0.1855, -0.0681, -0.0187]],
[[-0.0069, 0.0491, -0.1868], [[-0.0069, 0.0491, -0.1868],
[-0.1609, -0.0316, 0.0150], [-0.1609, -0.0316, 0.0150],
[-0.1605, 0.1506, -0.0074]], [-0.1605, 0.1506, -0.0074]],
[[ 0.0851, -0.1732, -0.1777], [[ 0.0851, -0.1732, -0.1777],
[ 0.0539, -0.0500, -0.1231], [ 0.0539, -0.0500, -0.1231],
[ 0.1654, 0.0342, -0.1904]]], [ 0.1654, 0.0342, -0.1904]]],
[[[ 0.0476, 0.0284, 0.1212], [[[ 0.0476, 0.0284, 0.1212],
[-0.1603, -0.1924, 0.0144], [-0.1603, -0.1924, 0.0144],
[ 0.0076, -0.0928, -0.1645]], [ 0.0076, -0.0928, -0.1645]],
[[ 0.0215, 0.1845, -0.1034], [[ 0.0215, 0.1845, -0.1034],
[ 0.1574, -0.1577, -0.0438], [ 0.1574, -0.1577, -0.0438],
[-0.1360, -0.0601, -0.1693]], [-0.1360, -0.0601, -0.1693]],
[[-0.0720, 0.0619, 0.1405], [[-0.0720, 0.0619, 0.1405],
[ 0.0699, -0.1288, 0.0041], [ 0.0699, -0.1288, 0.0041],
[-0.0381, -0.1697, -0.1568]]], [-0.0381, -0.1697, -0.1568]]],
[[[-0.1599, 0.1231, -0.1034], [[[-0.1599, 0.1231, -0.1034],
[-0.0314, 0.0105, -0.1449], [-0.0314, 0.0105, -0.1449],
[-0.0172, -0.0781, 0.0839]], [-0.0172, -0.0781, 0.0839]],
[[-0.0676, 0.1185, -0.1559], [[-0.0676, 0.1185, -0.1559],
[-0.1053, -0.1306, 0.1820], [-0.1053, -0.1306, 0.1820],
[ 0.1584, -0.1370, 0.1828]], [ 0.1584, -0.1370, 0.1828]],
[[ 0.0658, 0.1412, -0.0537], [[ 0.0658, 0.1412, -0.0537],
[-0.1230, -0.1411, -0.0011], [-0.1230, -0.1411, -0.0011],
[-0.1318, -0.0458, 0.1838]]], [-0.1318, -0.0458, 0.1838]]],
[[[-0.0268, 0.1747, -0.1037], [[[-0.0268, 0.1747, -0.1037],
[ 0.0515, -0.0228, -0.1024], [ 0.0515, -0.0228, -0.1024],
[-0.1543, -0.0643, -0.0100]], [-0.1543, -0.0643, -0.0100]],
[[-0.1572, -0.1530, 0.0026], [[-0.1572, -0.1530, 0.0026],
[ 0.1463, -0.1233, 0.0470], [ 0.1463, -0.1233, 0.0470],
[-0.1595, -0.1108, -0.0654]], [-0.1595, -0.1108, -0.0654]],
[[-0.0521, -0.0094, 0.1544], [[-0.0521, -0.0094, 0.1544],
[-0.0505, -0.0332, 0.0048], [-0.0505, -0.0332, 0.0048],
[ 0.0735, 0.1350, 0.0690]]], [ 0.0735, 0.1350, 0.0690]]],
[[[ 0.0025, 0.0724, 0.0930], [[[ 0.0025, 0.0724, 0.0930],
[-0.1885, 0.0475, 0.1100], [-0.1885, 0.0475, 0.1100],
[-0.1622, 0.0087, -0.0030]], [-0.1622, 0.0087, -0.0030]],
[[ 0.1032, -0.1425, -0.0620], [[ 0.1032, -0.1425, -0.0620],
[ 0.1515, -0.0736, -0.1888], [ 0.1515, -0.0736, -0.1888],
[-0.1246, 0.1424, -0.0491]], [-0.1246, 0.1424, -0.0491]],
[[ 0.1759, -0.1616, 0.1198], [[ 0.1759, -0.1616, 0.1198],
[-0.1103, 0.1032, 0.1727], [-0.1103, 0.1032, 0.1727],
[-0.0601, 0.1635, 0.0034]]], [-0.0601, 0.1635, 0.0034]]],
[[[ 0.0301, 0.1517, 0.0657], [[[ 0.0301, 0.1517, 0.0657],
[-0.1368, -0.1165, 0.1193], [-0.1368, -0.1165, 0.1193],
[-0.0962, 0.1451, 0.1099]], [-0.0962, 0.1451, 0.1099]],
[[ 0.1646, -0.1860, -0.1187], [[ 0.1646, -0.1860, -0.1187],
[-0.1367, -0.0911, 0.1337], [-0.1367, -0.0911, 0.1337],
[-0.0926, -0.0524, -0.0672]], [-0.0926, -0.0524, -0.0672]],
[[-0.1509, -0.1231, -0.0855], [[-0.1509, -0.1231, -0.0855],
[ 0.1808, -0.0713, 0.0410], [ 0.1808, -0.0713, 0.0410],
[-0.0621, -0.0506, 0.1871]]], [-0.0621, -0.0506, 0.1871]]],
[[[ 0.0888, -0.0874, -0.0826], [[[ 0.0888, -0.0874, -0.0826],
[ 0.0416, -0.0961, 0.0603], [ 0.0416, -0.0961, 0.0603],
[ 0.1455, 0.0050, 0.0318]], [ 0.1455, 0.0050, 0.0318]],
[[-0.1633, 0.0070, -0.1537], [[-0.1633, 0.0070, -0.1537],
[-0.0109, 0.1602, -0.0463], [-0.0109, 0.1602, -0.0463],
[-0.0423, -0.0147, -0.1045]], [-0.0423, -0.0147, -0.1045]],
[[ 0.1640, -0.0997, -0.1662], [[ 0.1640, -0.0997, -0.1662],
[-0.1074, 0.1549, -0.1905], [-0.1074, 0.1549, -0.1905],
[-0.1708, 0.1624, 0.0219]]], [-0.1708, 0.1624, 0.0219]]],
[[[ 0.0824, -0.1376, 0.1086], [[[ 0.0824, -0.1376, 0.1086],
[ 0.0836, 0.0135, 0.0351], [ 0.0836, 0.0135, 0.0351],
[-0.1518, 0.0784, -0.1708]], [-0.1518, 0.0784, -0.1708]],
[[-0.1636, -0.1571, 0.1032], [[-0.1636, -0.1571, 0.1032],
[-0.1152, 0.0274, -0.1022], [-0.1152, 0.0274, -0.1022],
[-0.0956, -0.1606, -0.1615]], [-0.0956, -0.1606, -0.1615]],
[[ 0.1307, 0.0419, 0.1924], [[ 0.1307, 0.0419, 0.1924],
[-0.0599, -0.1296, -0.0448], [-0.0599, -0.1296, -0.0448],
[ 0.0363, 0.0377, -0.0460]]], [ 0.0363, 0.0377, -0.0460]]],
[[[-0.1685, -0.1277, -0.0465], [[[-0.1685, -0.1277, -0.0465],
[ 0.0922, -0.1011, 0.0742], [ 0.0922, -0.1011, 0.0742],
[ 0.0053, -0.1456, 0.0135]], [ 0.0053, -0.1456, 0.0135]],
[[ 0.1341, 0.0131, 0.1281], [[ 0.1341, 0.0131, 0.1281],
[-0.1020, 0.1069, -0.0631], [-0.1020, 0.1069, -0.0631],
[-0.0439, -0.1189, -0.1822]], [-0.0439, -0.1189, -0.1822]],
[[ 0.1624, -0.1253, 0.0302], [[ 0.1624, -0.1253, 0.0302],
[ 0.0709, 0.0767, 0.1453], [ 0.0709, 0.0767, 0.1453],
[ 0.0203, 0.1603, -0.1720]]], [ 0.0203, 0.1603, -0.1720]]],
[[[-0.1550, 0.1513, -0.1003], [[[-0.1550, 0.1513, -0.1003],
[ 0.0370, 0.0367, -0.0233], [ 0.0370, 0.0367, -0.0233],
[ 0.0916, -0.0871, 0.1579]], [ 0.0916, -0.0871, 0.1579]],
[[-0.1900, 0.0314, 0.0865], [[-0.1900, 0.0314, 0.0865],
[-0.0197, 0.0296, -0.0048], [-0.0197, 0.0296, -0.0048],
[ 0.0846, 0.1543, -0.0770]], [ 0.0846, 0.1543, -0.0770]],
[[-0.0016, -0.0978, 0.1826], [[-0.0016, -0.0978, 0.1826],
[-0.0477, 0.0689, 0.1079], [-0.0477, 0.0689, 0.1079],
[ 0.0400, 0.0880, 0.1674]]], [ 0.0400, 0.0880, 0.1674]]],
[[[ 0.0145, -0.0447, -0.1742], [[[ 0.0145, -0.0447, -0.1742],
[ 0.0394, 0.0127, -0.1172], [ 0.0394, 0.0127, -0.1172],
[ 0.1330, -0.1207, 0.0326]], [ 0.1330, -0.1207, 0.0326]],
[[-0.0155, -0.1602, 0.0023], [[-0.0155, -0.1602, 0.0023],
[ 0.0789, 0.1648, 0.1781], [ 0.0789, 0.1648, 0.1781],
[-0.1468, -0.0481, -0.1260]], [-0.1468, -0.0481, -0.1260]],
[[-0.0139, 0.0848, -0.0536], [[-0.0139, 0.0848, -0.0536],
[-0.1581, 0.1130, 0.0717], [-0.1581, 0.1130, 0.0717],
[ 0.0275, -0.0006, -0.0049]]], [ 0.0275, -0.0006, -0.0049]]],
[[[-0.0199, 0.0032, -0.1246], [[[-0.0199, 0.0032, -0.1246],
[ 0.0479, 0.1418, -0.1295], [ 0.0479, 0.1418, -0.1295],
[-0.1646, -0.1139, -0.1018]], [-0.1646, -0.1139, -0.1018]],
[[ 0.1475, 0.1413, -0.0354], [[ 0.1475, 0.1413, -0.0354],
[ 0.0612, -0.1652, 0.0801], [ 0.0612, -0.1652, 0.0801],
[-0.1306, -0.0165, 0.1733]], [-0.1306, -0.0165, 0.1733]],
[[ 0.1527, 0.0911, -0.1906], [[ 0.1527, 0.0911, -0.1906],
[-0.1152, 0.1737, 0.0436], [-0.1152, 0.1737, 0.0436],
[-0.0213, -0.0314, -0.0319]]], [-0.0213, -0.0314, -0.0319]]],
[[[-0.0003, -0.0546, -0.1255], [[[-0.0003, -0.0546, -0.1255],
[ 0.0914, -0.1414, 0.0542], [ 0.0914, -0.1414, 0.0542],
[ 0.1139, 0.0132, 0.0815]], [ 0.1139, 0.0132, 0.0815]],
[[-0.0042, 0.0541, 0.1456], [[-0.0042, 0.0541, 0.1456],
[ 0.0509, -0.0790, 0.0272], [ 0.0509, -0.0790, 0.0272],
[ 0.1419, 0.0992, -0.1448]], [ 0.1419, 0.0992, -0.1448]],
[[ 0.0496, 0.0013, 0.0838], [[ 0.0496, 0.0013, 0.0838],
[-0.0662, 0.0315, -0.1168], [-0.0662, 0.0315, -0.1168],
[-0.0069, -0.1503, 0.0729]]], [-0.0069, -0.1503, 0.0729]]],
[[[ 0.1866, 0.1329, -0.0560], [[[ 0.1866, 0.1329, -0.0560],
[ 0.0026, 0.1533, 0.0326], [ 0.0026, 0.1533, 0.0326],
[-0.1161, -0.0323, 0.0053]], [-0.1161, -0.0323, 0.0053]],
[[-0.0243, -0.1823, -0.1657], [[-0.0243, -0.1823, -0.1657],
[-0.0107, -0.0832, 0.0029], [-0.0107, -0.0832, 0.0029],
[ 0.0981, 0.1241, -0.1788]], [ 0.0981, 0.1241, -0.1788]],
[[-0.0400, -0.0577, -0.0757], [[-0.0400, -0.0577, -0.0757],
[-0.0584, 0.0176, -0.1019], [-0.0584, 0.0176, -0.1019],
[-0.1828, 0.1589, -0.0312]]], [-0.1828, 0.1589, -0.0312]]],
[[[-0.1083, -0.1236, -0.0904], [[[-0.1083, -0.1236, -0.0904],
[-0.1575, 0.0157, 0.0552], [-0.1575, 0.0157, 0.0552],
[-0.0839, 0.1704, -0.1457]], [-0.0839, 0.1704, -0.1457]],
[[-0.1648, -0.0270, -0.0489], [[-0.1648, -0.0270, -0.0489],
[-0.1122, -0.0288, -0.0073], [-0.1122, -0.0288, -0.0073],
[-0.1443, -0.1712, 0.0100]], [-0.1443, -0.1712, 0.0100]],
[[-0.1142, -0.1552, 0.1568], [[-0.1142, -0.1552, 0.1568],
[ 0.0743, -0.1108, -0.0643], [ 0.0743, -0.1108, -0.0643],
[-0.0394, -0.1345, 0.0992]]], [-0.0394, -0.1345, 0.0992]]],
[[[-0.1591, 0.0942, -0.1035], [[[-0.1591, 0.0942, -0.1035],
[-0.0781, 0.0725, -0.0888], [-0.0781, 0.0725, -0.0888],
[ 0.0959, 0.0213, 0.1222]], [ 0.0959, 0.0213, 0.1222]],
[[ 0.1202, -0.0217, -0.0955], [[ 0.1202, -0.0217, -0.0955],
[-0.1748, -0.1133, -0.0704], [-0.1748, -0.1133, -0.0704],
[-0.0670, -0.1401, 0.1553]], [-0.0670, -0.1401, 0.1553]],
[[ 0.0053, -0.0871, -0.0239], [[ 0.0053, -0.0871, -0.0239],
[ 0.0961, -0.0547, 0.1741], [ 0.0961, -0.0547, 0.1741],
[-0.0570, 0.0477, 0.1853]]], [-0.0570, 0.0477, 0.1853]]],
[[[-0.1115, -0.0183, -0.1302], [[[-0.1115, -0.0183, -0.1302],
[ 0.1435, -0.0238, -0.0048], [ 0.1435, -0.0238, -0.0048],
[ 0.1862, -0.1837, 0.1711]], [ 0.1862, -0.1837, 0.1711]],
[[ 0.1375, -0.1798, 0.0818], [[ 0.1375, -0.1798, 0.0818],
[-0.0792, 0.0820, 0.1373], [-0.0792, 0.0820, 0.1373],
[ 0.1849, 0.0672, -0.1822]], [ 0.1849, 0.0672, -0.1822]],
[[ 0.1868, -0.0356, 0.0726], [[ 0.1868, -0.0356, 0.0726],
[-0.1523, -0.1130, 0.1506], [-0.1523, -0.1130, 0.1506],
[-0.1046, 0.0178, 0.0990]]], [-0.1046, 0.0178, 0.0990]]],
[[[ 0.1321, -0.1641, 0.0411], [[[ 0.1321, -0.1641, 0.0411],
[ 0.0526, 0.0393, 0.0918], [ 0.0526, 0.0393, 0.0918],
[-0.1345, 0.0750, 0.0859]], [-0.1345, 0.0750, 0.0859]],
[[-0.0985, 0.1466, 0.1349], [[-0.0985, 0.1466, 0.1349],
[-0.1461, -0.1742, 0.0941], [-0.1461, -0.1742, 0.0941],
[-0.1502, -0.1813, 0.0864]], [-0.1502, -0.1813, 0.0864]],
[[-0.1039, 0.1179, 0.1499], [[-0.1039, 0.1179, 0.1499],
[-0.0366, -0.0120, 0.0951], [-0.0366, -0.0120, 0.0951],
[ 0.0087, 0.1212, -0.0183]]], [ 0.0087, 0.1212, -0.0183]]],
[[[-0.1375, 0.0765, -0.0072], [[[-0.1375, 0.0765, -0.0072],
[-0.0041, 0.0379, -0.0243], [-0.0041, 0.0379, -0.0243],
[-0.1495, 0.1601, 0.1575]], [-0.1495, 0.1601, 0.1575]],
[[-0.0454, 0.1642, 0.0720], [[-0.0454, 0.1642, 0.0720],
[-0.0533, 0.0150, 0.0039], [-0.0533, 0.0150, 0.0039],
[ 0.0194, 0.0113, -0.1194]], [ 0.0194, 0.0113, -0.1194]],
[[ 0.0527, -0.0886, 0.0359], [[ 0.0527, -0.0886, 0.0359],
[ 0.1595, 0.0526, -0.0048], [ 0.1595, 0.0526, -0.0048],
[-0.1790, -0.0458, -0.0324]]], [-0.1790, -0.0458, -0.0324]]],
[[[-0.1638, 0.0942, 0.0686], [[[-0.1638, 0.0942, 0.0686],
[-0.1082, -0.0675, 0.1892], [-0.1082, -0.0675, 0.1892],
[-0.1347, -0.1247, 0.0739]], [-0.1347, -0.1247, 0.0739]],
[[ 0.0595, 0.1504, -0.1657], [[ 0.0595, 0.1504, -0.1657],
[ 0.0733, 0.0529, -0.1599], [ 0.0733, 0.0529, -0.1599],
[ 0.0171, -0.1127, -0.0259]], [ 0.0171, -0.1127, -0.0259]],
[[-0.0092, 0.0193, 0.1176], [[-0.0092, 0.0193, 0.1176],
[-0.1183, 0.0101, 0.1011], [-0.1183, 0.0101, 0.1011],
[ 0.0648, -0.1897, 0.0782]]], [ 0.0648, -0.1897, 0.0782]]],
[[[ 0.0098, -0.1161, -0.0802], [[[ 0.0098, -0.1161, -0.0802],
[-0.1821, 0.0221, -0.1754], [-0.1821, 0.0221, -0.1754],
[-0.1218, 0.0525, -0.0480]], [-0.1218, 0.0525, -0.0480]],
[[ 0.0770, 0.0477, 0.1514], [[ 0.0770, 0.0477, 0.1514],
[ 0.0374, -0.1075, -0.1026], [ 0.0374, -0.1075, -0.1026],
[-0.0581, -0.1011, 0.1241]], [-0.0581, -0.1011, 0.1241]],
[[-0.0567, -0.0163, 0.0374], [[-0.0567, -0.0163, 0.0374],
[-0.1739, -0.0579, 0.0704], [-0.1739, -0.0579, 0.0704],
[ 0.1817, 0.1561, 0.1677]]], [ 0.1817, 0.1561, 0.1677]]],
[[[-0.0569, -0.0763, 0.0044], [[[-0.0569, -0.0763, 0.0044],
[-0.1133, 0.0813, 0.1477], [-0.1133, 0.0813, 0.1477],
[ 0.0836, 0.0483, -0.1800]], [ 0.0836, 0.0483, -0.1800]],
[[ 0.1343, -0.1590, 0.1177], [[ 0.1343, -0.1590, 0.1177],
[ 0.1071, -0.1647, -0.0646], [ 0.1071, -0.1647, -0.0646],
[ 0.1578, -0.1261, 0.0243]], [ 0.1578, -0.1261, 0.0243]],
[[-0.0424, -0.0241, -0.0988], [[-0.0424, -0.0241, -0.0988],
[ 0.0023, 0.0029, -0.0291], [ 0.0023, 0.0029, -0.0291],
[ 0.0415, -0.0557, 0.1427]]], [ 0.0415, -0.0557, 0.1427]]],
[[[-0.1028, 0.1054, 0.1658], [[[-0.1028, 0.1054, 0.1658],
[-0.0357, 0.1579, 0.1237], [-0.0357, 0.1579, 0.1237],
[ 0.0368, 0.0532, -0.1043]], [ 0.0368, 0.0532, -0.1043]],
[[-0.0369, 0.0575, -0.1023], [[-0.0369, 0.0575, -0.1023],
[ 0.0635, 0.1015, 0.1112], [ 0.0635, 0.1015, 0.1112],
[-0.1235, 0.0467, 0.0908]], [-0.1235, 0.0467, 0.0908]],
[[ 0.1380, 0.0633, 0.1087], [[ 0.1380, 0.0633, 0.1087],
[-0.1360, 0.0422, -0.1524], [-0.1360, 0.0422, -0.1524],
[ 0.0819, 0.0918, -0.1624]]], [ 0.0819, 0.0918, -0.1624]]],
[[[ 0.1584, -0.0218, -0.0236], [[[ 0.1584, -0.0218, -0.0236],
[ 0.1878, -0.1289, 0.1343], [ 0.1878, -0.1289, 0.1343],
[ 0.0351, 0.1225, -0.1460]], [ 0.0351, 0.1225, -0.1460]],
[[ 0.0690, -0.1439, 0.0056], [[ 0.0690, -0.1439, 0.0056],
[ 0.0272, -0.0058, 0.0125], [ 0.0272, -0.0058, 0.0125],
[ 0.0868, -0.0684, -0.0884]], [ 0.0868, -0.0684, -0.0884]],
[[ 0.1045, 0.0583, -0.0870], [[ 0.1045, 0.0583, -0.0870],
[ 0.0600, -0.0732, -0.1695], [ 0.0600, -0.0732, -0.1695],
[ 0.0953, 0.0246, 0.1245]]]])), [ 0.0953, 0.0246, 0.1245]]]])),
('conv1.bias', ('conv1.bias',
tensor([-1.8698e-01, -7.9379e-06, -1.9277e-02, 5.2182e-02, 7.5716e-02, tensor([-1.8698e-01, -7.9379e-06, -1.9277e-02, 5.2182e-02, 7.5716e-02,
-3.3830e-03, -9.6565e-02, 1.0241e-01, -8.2457e-02, -1.6224e-01, -3.3830e-03, -9.6565e-02, 1.0241e-01, -8.2457e-02, -1.6224e-01,
1.2980e-01, -8.2256e-02, -7.4655e-02, -3.7980e-02, 8.3407e-02, 1.2980e-01, -8.2256e-02, -7.4655e-02, -3.7980e-02, 8.3407e-02,
-1.4880e-01, 4.8939e-02, 2.7506e-02, 5.8676e-03, -1.5813e-01, -1.4880e-01, 4.8939e-02, 2.7506e-02, 5.8676e-03, -1.5813e-01,
-6.2464e-04, 1.0359e-02, -1.5525e-01, 7.9100e-02, 1.6850e-02, -6.2464e-04, 1.0359e-02, -1.5525e-01, 7.9100e-02, 1.6850e-02,
-1.3809e-01, -6.3393e-02, -5.3843e-02, -1.5219e-02, -1.7365e-01, -1.3809e-01, -6.3393e-02, -5.3843e-02, -1.5219e-02, -1.7365e-01,
1.7249e-01, -1.1165e-01])), 1.7249e-01, -1.1165e-01])),
('conv2.weight', ('conv2.weight',
tensor([[[[ 4.3750e-02, 4.5533e-02, -2.9410e-02], tensor([[[[ 4.3750e-02, 4.5533e-02, -2.9410e-02],
[-4.1395e-02, 5.0397e-04, -1.3265e-02], [-4.1395e-02, 5.0397e-04, -1.3265e-02],
[-4.9851e-02, -1.0518e-02, 5.7710e-02]], [-4.9851e-02, -1.0518e-02, 5.7710e-02]],
[[-5.6332e-02, -4.7168e-03, -4.4627e-02], [[-5.6332e-02, -4.7168e-03, -4.4627e-02],
[ 5.3513e-03, -4.0824e-02, 1.8281e-02], [ 5.3513e-03, -4.0824e-02, 1.8281e-02],
[ 5.0677e-02, -1.5295e-02, -6.1751e-03]], [ 5.0677e-02, -1.5295e-02, -6.1751e-03]],
[[-2.4984e-02, 1.2784e-02, -4.7123e-02], [[-2.4984e-02, 1.2784e-02, -4.7123e-02],
[-4.3238e-02, 4.7349e-02, -1.5219e-02], [-4.3238e-02, 4.7349e-02, -1.5219e-02],
[-3.6073e-02, 4.1506e-02, -3.5337e-02]], [-3.6073e-02, 4.1506e-02, -3.5337e-02]],
..., ...,
[[ 5.6048e-02, 1.9129e-03, -1.7200e-02], [[ 5.6048e-02, 1.9129e-03, -1.7200e-02],
[ 5.8869e-02, -5.1520e-02, -5.3205e-02], [ 5.8869e-02, -5.1520e-02, -5.3205e-02],
[-1.3903e-02, 5.1790e-02, 2.2585e-02]], [-1.3903e-02, 5.1790e-02, 2.2585e-02]],
[[ 1.1835e-02, -4.9313e-02, -3.1838e-02], [[ 1.1835e-02, -4.9313e-02, -3.1838e-02],
[ 7.6813e-03, 4.2715e-02, -5.7404e-02], [ 7.6813e-03, 4.2715e-02, -5.7404e-02],
[-4.1474e-02, -2.3128e-02, -4.7935e-02]], [-4.1474e-02, -2.3128e-02, -4.7935e-02]],
[[-2.1860e-02, -2.1817e-02, -3.2578e-02], [[-2.1860e-02, -2.1817e-02, -3.2578e-02],
[ 3.1317e-02, 3.3435e-02, 3.1837e-02], [ 3.1317e-02, 3.3435e-02, 3.1837e-02],
[-2.2399e-03, 3.1600e-02, 4.0183e-02]]], [-2.2399e-03, 3.1600e-02, 4.0183e-02]]],
[[[ 1.9610e-04, 5.3780e-02, -4.5810e-02], [[[ 1.9610e-04, 5.3780e-02, -4.5810e-02],
[ 4.0340e-02, 1.4904e-02, -1.5597e-02], [ 4.0340e-02, 1.4904e-02, -1.5597e-02],
[-4.6080e-02, 5.0714e-02, -5.7445e-03]], [-4.6080e-02, 5.0714e-02, -5.7445e-03]],
[[-3.5281e-02, 3.3011e-02, 4.3343e-02], [[-3.5281e-02, 3.3011e-02, 4.3343e-02],
[-4.6263e-02, -5.6184e-02, 5.1245e-03], [-4.6263e-02, -5.6184e-02, 5.1245e-03],
[ 3.6015e-02, -3.3152e-02, 4.6629e-03]], [ 3.6015e-02, -3.3152e-02, 4.6629e-03]],
[[ 1.7650e-03, -4.2336e-02, 4.3744e-02], [[ 1.7650e-03, -4.2336e-02, 4.3744e-02],
[ 2.1655e-02, 5.3759e-02, 1.3719e-03], [ 2.1655e-02, 5.3759e-02, 1.3719e-03],
[ 4.2005e-02, 5.3998e-02, 1.9009e-02]], [ 4.2005e-02, 5.3998e-02, 1.9009e-02]],
..., ...,
[[ 8.9786e-03, -1.8645e-02, -1.3587e-02], [[ 8.9786e-03, -1.8645e-02, -1.3587e-02],
[-5.4167e-02, 1.2335e-02, -3.0384e-02], [-5.4167e-02, 1.2335e-02, -3.0384e-02],
[-4.8722e-03, -3.7296e-02, -2.6446e-02]], [-4.8722e-03, -3.7296e-02, -2.6446e-02]],
[[ 1.7580e-02, 3.8462e-02, -5.0269e-02], [[ 1.7580e-02, 3.8462e-02, -5.0269e-02],
[ 2.6601e-03, -1.1462e-02, 4.7459e-02], [ 2.6601e-03, -1.1462e-02, 4.7459e-02],
[-2.8888e-02, 3.4436e-02, -4.9943e-02]], [-2.8888e-02, 3.4436e-02, -4.9943e-02]],
[[-5.0206e-02, -5.6025e-02, -3.6346e-02], [[-5.0206e-02, -5.6025e-02, -3.6346e-02],
[-2.4407e-02, 5.3721e-02, -5.4920e-02], [-2.4407e-02, 5.3721e-02, -5.4920e-02],
[ 5.1835e-02, -3.2396e-02, 3.2373e-02]]], [ 5.1835e-02, -3.2396e-02, 3.2373e-02]]],
[[[-2.7759e-02, -5.4737e-02, -1.1689e-02], [[[-2.7759e-02, -5.4737e-02, -1.1689e-02],
[ 3.9462e-02, 2.8649e-02, 5.1776e-02], [ 3.9462e-02, 2.8649e-02, 5.1776e-02],
[ 2.4253e-02, -2.8318e-02, 2.7402e-02]], [ 2.4253e-02, -2.8318e-02, 2.7402e-02]],
[[ 1.3045e-02, -1.0456e-02, 2.0426e-02], [[ 1.3045e-02, -1.0456e-02, 2.0426e-02],
[ 2.1949e-02, 4.6817e-02, -5.6093e-02], [ 2.1949e-02, 4.6817e-02, -5.6093e-02],
[ 2.7145e-02, -5.5441e-02, -2.0719e-02]], [ 2.7145e-02, -5.5441e-02, -2.0719e-02]],
[[ 4.4704e-02, -2.4099e-02, -4.7185e-02], [[ 4.4704e-02, -2.4099e-02, -4.7185e-02],
[-4.3257e-02, -3.3058e-02, -8.6451e-03], [-4.3257e-02, -3.3058e-02, -8.6451e-03],
[-3.7283e-02, -3.4569e-02, -7.1049e-03]], [-3.7283e-02, -3.4569e-02, -7.1049e-03]],
..., ...,
[[-4.1559e-02, -2.9240e-02, 2.7197e-03], [[-4.1559e-02, -2.9240e-02, 2.7197e-03],
[ 2.0770e-02, 5.4479e-02, -4.4845e-02], [ 2.0770e-02, 5.4479e-02, -4.4845e-02],
[-1.1641e-02, -2.9814e-02, -2.4419e-02]], [-1.1641e-02, -2.9814e-02, -2.4419e-02]],
[[-1.5743e-02, 1.0854e-02, 3.0878e-02], [[-1.5743e-02, 1.0854e-02, 3.0878e-02],
[ 2.2739e-02, 3.2999e-02, -1.1902e-02], [ 2.2739e-02, 3.2999e-02, -1.1902e-02],
[-3.4837e-02, 1.5305e-02, -8.7552e-03]], [-3.4837e-02, 1.5305e-02, -8.7552e-03]],
[[-2.2882e-02, 9.4639e-03, 5.1878e-03], [[-2.2882e-02, 9.4639e-03, 5.1878e-03],
[-2.6344e-02, 2.9063e-02, -1.9337e-02], [-2.6344e-02, 2.9063e-02, -1.9337e-02],
[-3.4314e-02, 1.5313e-02, 4.1524e-02]]], [-3.4314e-02, 1.5313e-02, 4.1524e-02]]],
..., ...,
[[[ 9.7454e-03, -3.2903e-03, 1.0696e-02], [[[ 9.7454e-03, -3.2903e-03, 1.0696e-02],
[-4.0918e-02, 1.6352e-02, 1.4646e-02], [-4.0918e-02, 1.6352e-02, 1.4646e-02],
[ 1.2516e-02, -2.1804e-02, -2.5489e-02]], [ 1.2516e-02, -2.1804e-02, -2.5489e-02]],
[[-1.6083e-02, 2.5374e-02, 3.1458e-02], [[-1.6083e-02, 2.5374e-02, 3.1458e-02],
[-3.1497e-02, -1.9513e-02, -2.1223e-02], [-3.1497e-02, -1.9513e-02, -2.1223e-02],
[ 6.6286e-03, 1.6538e-02, -4.8944e-02]], [ 6.6286e-03, 1.6538e-02, -4.8944e-02]],
[[ 2.4808e-02, -2.9520e-02, -4.8227e-02], [[ 2.4808e-02, -2.9520e-02, -4.8227e-02],
[ 1.7325e-03, -4.7443e-02, 2.3087e-03], [ 1.7325e-03, -4.7443e-02, 2.3087e-03],
[-1.0008e-02, -2.0313e-02, 2.9944e-02]], [-1.0008e-02, -2.0313e-02, 2.9944e-02]],
..., ...,
[[ 5.5781e-02, -2.0898e-02, -3.1487e-03], [[ 5.5781e-02, -2.0898e-02, -3.1487e-03],
[-1.6931e-02, 4.5279e-04, -1.5024e-02], [-1.6931e-02, 4.5279e-04, -1.5024e-02],
[-5.5885e-02, 2.7140e-02, -8.5434e-03]], [-5.5885e-02, 2.7140e-02, -8.5434e-03]],
[[ 1.3970e-02, -3.3131e-02, 4.3112e-02], [[ 1.3970e-02, -3.3131e-02, 4.3112e-02],
[-3.4956e-02, -5.0144e-02, -1.6391e-02], [-3.4956e-02, -5.0144e-02, -1.6391e-02],
[-9.1003e-03, -2.0204e-02, -1.0226e-03]], [-9.1003e-03, -2.0204e-02, -1.0226e-03]],
[[-4.0053e-02, -5.0194e-02, 5.0405e-02], [[-4.0053e-02, -5.0194e-02, 5.0405e-02],
[ 5.4107e-02, 4.2185e-02, 3.4359e-02], [ 5.4107e-02, 4.2185e-02, 3.4359e-02],
[ 1.6749e-02, -1.4102e-02, 5.0171e-02]]], [ 1.6749e-02, -1.4102e-02, 5.0171e-02]]],
[[[ 2.8229e-02, 5.6586e-02, -3.9617e-03], [[[ 2.8229e-02, 5.6586e-02, -3.9617e-03],
[ 2.9538e-02, -1.2507e-02, -2.5516e-02], [ 2.9538e-02, -1.2507e-02, -2.5516e-02],
[-1.5193e-02, -2.9232e-02, -2.0701e-02]], [-1.5193e-02, -2.9232e-02, -2.0701e-02]],
[[-5.8773e-02, 3.3015e-02, -9.4146e-03], [[-5.8773e-02, 3.3015e-02, -9.4146e-03],
[ 2.8957e-02, 5.8666e-02, 2.8679e-02], [ 2.8957e-02, 5.8666e-02, 2.8679e-02],
[ 1.5249e-02, -1.2246e-03, 1.2230e-03]], [ 1.5249e-02, -1.2246e-03, 1.2230e-03]],
[[ 2.6050e-02, -4.6042e-02, -3.4895e-03], [[ 2.6050e-02, -4.6042e-02, -3.4895e-03],
[ 4.9529e-02, 6.6835e-03, -4.1808e-02], [ 4.9529e-02, 6.6835e-03, -4.1808e-02],
[-8.6450e-03, -4.8510e-02, -2.4011e-02]], [-8.6450e-03, -4.8510e-02, -2.4011e-02]],
..., ...,
[[-5.1427e-02, 2.4528e-02, -5.4878e-02], [[-5.1427e-02, 2.4528e-02, -5.4878e-02],
[-1.8610e-02, 5.4365e-02, 3.5053e-03], [-1.8610e-02, 5.4365e-02, 3.5053e-03],
[-3.9922e-02, 4.2510e-02, -5.7261e-02]], [-3.9922e-02, 4.2510e-02, -5.7261e-02]],
[[ 4.1938e-02, -4.2039e-02, -1.2487e-02], [[ 4.1938e-02, -4.2039e-02, -1.2487e-02],
[-1.4090e-02, -3.7895e-02, 1.4394e-02], [-1.4090e-02, -3.7895e-02, 1.4394e-02],
[ 2.2555e-02, -2.7264e-02, 5.6102e-02]], [ 2.2555e-02, -2.7264e-02, 5.6102e-02]],
[[ 1.5770e-02, 5.4672e-02, -2.4056e-02], [[ 1.5770e-02, 5.4672e-02, -2.4056e-02],
[ 5.2089e-02, -2.8859e-02, -2.6499e-03], [ 5.2089e-02, -2.8859e-02, -2.6499e-03],
[-5.2122e-02, -3.7436e-02, 3.9897e-02]]], [-5.2122e-02, -3.7436e-02, 3.9897e-02]]],
[[[ 2.7888e-02, 2.9241e-02, -1.9488e-02], [[[ 2.7888e-02, 2.9241e-02, -1.9488e-02],
[ 2.8928e-02, 5.3312e-02, -2.9810e-02], [ 2.8928e-02, 5.3312e-02, -2.9810e-02],
[-8.5104e-03, 5.7751e-02, -8.1857e-03]], [-8.5104e-03, 5.7751e-02, -8.1857e-03]],
[[ 4.2649e-02, 3.3158e-03, 4.2879e-02], [[ 4.2649e-02, 3.3158e-03, 4.2879e-02],
[ 7.7893e-03, -3.2879e-02, 2.7630e-02], [ 7.7893e-03, -3.2879e-02, 2.7630e-02],
[ 5.4706e-03, 4.8019e-02, 1.2420e-02]], [ 5.4706e-03, 4.8019e-02, 1.2420e-02]],
[[-4.2004e-02, -4.2790e-02, 2.4634e-02], [[-4.2004e-02, -4.2790e-02, 2.4634e-02],
[-5.4641e-02, 3.4600e-02, 2.9071e-03], [-5.4641e-02, 3.4600e-02, 2.9071e-03],
[ 2.6470e-02, 4.6701e-02, 3.7158e-02]], [ 2.6470e-02, 4.6701e-02, 3.7158e-02]],
..., ...,
[[-1.7641e-02, -2.1205e-02, -5.1504e-02], [[-1.7641e-02, -2.1205e-02, -5.1504e-02],
[-7.4737e-03, 5.5061e-02, -2.6397e-03], [-7.4737e-03, 5.5061e-02, -2.6397e-03],
[-4.4653e-02, -3.6719e-02, 3.4420e-06]], [-4.4653e-02, -3.6719e-02, 3.4420e-06]],
[[ 1.6525e-02, 1.7280e-02, 5.4554e-03], [[ 1.6525e-02, 1.7280e-02, 5.4554e-03],
[ 4.0098e-02, 2.7571e-02, -4.4965e-02], [ 4.0098e-02, 2.7571e-02, -4.4965e-02],
[ 6.1493e-03, -5.7754e-02, 1.0513e-02]], [ 6.1493e-03, -5.7754e-02, 1.0513e-02]],
[[-5.7615e-02, 3.2921e-02, -1.5900e-02], [[-5.7615e-02, 3.2921e-02, -1.5900e-02],
[ 2.0081e-02, 5.4590e-02, 1.1296e-02], [ 2.0081e-02, 5.4590e-02, 1.1296e-02],
[-4.5015e-02, 1.1341e-03, 2.6447e-02]]]])), [-4.5015e-02, 1.1341e-03, 2.6447e-02]]]])),
('conv2.bias', ('conv2.bias',
tensor([-0.0538, -0.0320, -0.0153, 0.0558, 0.0254, 0.0281, -0.0148, 0.0060, tensor([-0.0538, -0.0320, -0.0153, 0.0558, 0.0254, 0.0281, -0.0148, 0.0060,
-0.0283, -0.0062, 0.0437, -0.0064, 0.0341, 0.0233, -0.0201, 0.0391, -0.0283, -0.0062, 0.0437, -0.0064, 0.0341, 0.0233, -0.0201, 0.0391,
0.0243, 0.0071, 0.0125, -0.0138, -0.0377, -0.0169, -0.0475, -0.0004, 0.0243, 0.0071, 0.0125, -0.0138, -0.0377, -0.0169, -0.0475, -0.0004,
-0.0105, -0.0502, 0.0241, 0.0090, 0.0069, -0.0315, -0.0192, 0.0204])), -0.0105, -0.0502, 0.0241, 0.0090, 0.0069, -0.0315, -0.0192, 0.0204])),
('conv3.weight', ('conv3.weight',
tensor([[[[-0.0215, -0.0208, -0.0272], tensor([[[[-0.0215, -0.0208, -0.0272],
[-0.0493, -0.0117, -0.0285], [-0.0493, -0.0117, -0.0285],
[ 0.0515, -0.0041, 0.0126]], [ 0.0515, -0.0041, 0.0126]],
[[ 0.0299, -0.0301, 0.0552], [[ 0.0299, -0.0301, 0.0552],
[ 0.0450, 0.0449, -0.0583], [ 0.0450, 0.0449, -0.0583],
[-0.0452, -0.0480, -0.0275]], [-0.0452, -0.0480, -0.0275]],
[[-0.0262, -0.0338, 0.0505], [[-0.0262, -0.0338, 0.0505],
[ 0.0146, -0.0364, -0.0044], [ 0.0146, -0.0364, -0.0044],
[-0.0102, -0.0051, 0.0017]], [-0.0102, -0.0051, 0.0017]],
..., ...,
[[-0.0367, -0.0468, -0.0586], [[-0.0367, -0.0468, -0.0586],
[ 0.0126, 0.0037, 0.0191], [ 0.0126, 0.0037, 0.0191],
[-0.0153, 0.0048, -0.0160]], [-0.0153, 0.0048, -0.0160]],
[[-0.0050, 0.0364, 0.0582], [[-0.0050, 0.0364, 0.0582],
[ 0.0093, -0.0268, -0.0355], [ 0.0093, -0.0268, -0.0355],
[-0.0125, 0.0500, 0.0009]], [-0.0125, 0.0500, 0.0009]],
[[ 0.0237, -0.0211, -0.0130], [[ 0.0237, -0.0211, -0.0130],
[-0.0489, 0.0118, 0.0387], [-0.0489, 0.0118, 0.0387],
[-0.0006, 0.0301, 0.0283]]], [-0.0006, 0.0301, 0.0283]]],
[[[-0.0391, -0.0464, -0.0158], [[[-0.0391, -0.0464, -0.0158],
[ 0.0201, -0.0054, 0.0422], [ 0.0201, -0.0054, 0.0422],
[ 0.0085, -0.0474, -0.0251]], [ 0.0085, -0.0474, -0.0251]],
[[-0.0346, 0.0536, -0.0391], [[-0.0346, 0.0536, -0.0391],
[ 0.0244, -0.0263, -0.0073], [ 0.0244, -0.0263, -0.0073],
[ 0.0076, 0.0160, 0.0044]], [ 0.0076, 0.0160, 0.0044]],
[[-0.0128, 0.0146, -0.0381], [[-0.0128, 0.0146, -0.0381],
[-0.0277, -0.0142, 0.0226], [-0.0277, -0.0142, 0.0226],
[ 0.0190, 0.0326, -0.0219]], [ 0.0190, 0.0326, -0.0219]],
..., ...,
[[-0.0217, 0.0129, 0.0558], [[-0.0217, 0.0129, 0.0558],
[ 0.0164, -0.0292, -0.0467], [ 0.0164, -0.0292, -0.0467],
[-0.0296, 0.0205, -0.0300]], [-0.0296, 0.0205, -0.0300]],
[[ 0.0254, -0.0151, -0.0583], [[ 0.0254, -0.0151, -0.0583],
[ 0.0111, -0.0469, -0.0300], [ 0.0111, -0.0469, -0.0300],
[-0.0462, 0.0293, -0.0351]], [-0.0462, 0.0293, -0.0351]],
[[ 0.0401, -0.0251, 0.0160], [[ 0.0401, -0.0251, 0.0160],
[-0.0160, -0.0195, -0.0065], [-0.0160, -0.0195, -0.0065],
[-0.0519, 0.0351, 0.0357]]], [-0.0519, 0.0351, 0.0357]]],
[[[ 0.0544, -0.0209, -0.0454], [[[ 0.0544, -0.0209, -0.0454],
[ 0.0287, -0.0205, -0.0294], [ 0.0287, -0.0205, -0.0294],
[-0.0195, -0.0235, -0.0378]], [-0.0195, -0.0235, -0.0378]],
[[-0.0294, -0.0380, 0.0301], [[-0.0294, -0.0380, 0.0301],
[ 0.0360, 0.0367, 0.0458], [ 0.0360, 0.0367, 0.0458],
[-0.0189, -0.0017, 0.0145]], [-0.0189, -0.0017, 0.0145]],
[[-0.0297, 0.0567, 0.0276], [[-0.0297, 0.0567, 0.0276],
[ 0.0298, 0.0383, 0.0227], [ 0.0298, 0.0383, 0.0227],
[ 0.0262, 0.0063, 0.0131]], [ 0.0262, 0.0063, 0.0131]],
..., ...,
[[ 0.0002, 0.0432, -0.0247], [[ 0.0002, 0.0432, -0.0247],
[ 0.0068, -0.0298, -0.0484], [ 0.0068, -0.0298, -0.0484],
[-0.0361, -0.0014, 0.0444]], [-0.0361, -0.0014, 0.0444]],
[[-0.0184, -0.0201, -0.0163], [[-0.0184, -0.0201, -0.0163],
[-0.0466, 0.0255, -0.0244], [-0.0466, 0.0255, -0.0244],
[ 0.0283, 0.0149, -0.0588]], [ 0.0283, 0.0149, -0.0588]],
[[ 0.0323, 0.0392, -0.0254], [[ 0.0323, 0.0392, -0.0254],
[ 0.0560, 0.0137, -0.0401], [ 0.0560, 0.0137, -0.0401],
[-0.0236, 0.0589, 0.0448]]], [-0.0236, 0.0589, 0.0448]]],
..., ...,
[[[ 0.0213, 0.0204, 0.0574], [[[ 0.0213, 0.0204, 0.0574],
[-0.0276, -0.0196, 0.0117], [-0.0276, -0.0196, 0.0117],
[ 0.0569, -0.0158, -0.0502]], [ 0.0569, -0.0158, -0.0502]],
[[ 0.0452, -0.0038, 0.0502], [[ 0.0452, -0.0038, 0.0502],
[ 0.0428, -0.0398, -0.0486], [ 0.0428, -0.0398, -0.0486],
[ 0.0130, 0.0563, 0.0576]], [ 0.0130, 0.0563, 0.0576]],
[[ 0.0484, -0.0535, 0.0048], [[ 0.0484, -0.0535, 0.0048],
[ 0.0268, -0.0290, -0.0390], [ 0.0268, -0.0290, -0.0390],
[ 0.0189, -0.0194, -0.0588]], [ 0.0189, -0.0194, -0.0588]],
..., ...,
[[ 0.0163, -0.0113, -0.0520], [[ 0.0163, -0.0113, -0.0520],
[ 0.0288, -0.0547, -0.0544], [ 0.0288, -0.0547, -0.0544],
[ 0.0442, 0.0376, 0.0566]], [ 0.0442, 0.0376, 0.0566]],
[[-0.0343, 0.0569, 0.0438], [[-0.0343, 0.0569, 0.0438],
[-0.0403, -0.0372, -0.0532], [-0.0403, -0.0372, -0.0532],
[ 0.0322, 0.0126, 0.0423]], [ 0.0322, 0.0126, 0.0423]],
[[ 0.0577, 0.0136, -0.0480], [[ 0.0577, 0.0136, -0.0480],
[-0.0293, -0.0348, 0.0342], [-0.0293, -0.0348, 0.0342],
[-0.0510, -0.0078, -0.0042]]], [-0.0510, -0.0078, -0.0042]]],
[[[-0.0243, 0.0406, 0.0537], [[[-0.0243, 0.0406, 0.0537],
[ 0.0209, -0.0059, -0.0487], [ 0.0209, -0.0059, -0.0487],
[-0.0425, 0.0339, 0.0444]], [-0.0425, 0.0339, 0.0444]],
[[ 0.0465, -0.0467, 0.0461], [[ 0.0465, -0.0467, 0.0461],
[-0.0389, 0.0144, -0.0502], [-0.0389, 0.0144, -0.0502],
[ 0.0274, 0.0552, 0.0356]], [ 0.0274, 0.0552, 0.0356]],
[[-0.0289, 0.0474, -0.0217], [[-0.0289, 0.0474, -0.0217],
[ 0.0472, -0.0135, 0.0164], [ 0.0472, -0.0135, 0.0164],
[-0.0165, -0.0049, -0.0475]], [-0.0165, -0.0049, -0.0475]],
..., ...,
[[-0.0377, 0.0267, 0.0367], [[-0.0377, 0.0267, 0.0367],
[ 0.0111, 0.0114, -0.0329], [ 0.0111, 0.0114, -0.0329],
[ 0.0031, -0.0223, -0.0280]], [ 0.0031, -0.0223, -0.0280]],
[[-0.0500, -0.0529, 0.0116], [[-0.0500, -0.0529, 0.0116],
[ 0.0483, 0.0121, -0.0149], [ 0.0483, 0.0121, -0.0149],
[ 0.0328, 0.0201, 0.0402]], [ 0.0328, 0.0201, 0.0402]],
[[ 0.0463, 0.0157, -0.0332], [[ 0.0463, 0.0157, -0.0332],
[ 0.0150, 0.0479, 0.0461], [ 0.0150, 0.0479, 0.0461],
[ 0.0275, 0.0506, -0.0466]]], [ 0.0275, 0.0506, -0.0466]]],
[[[-0.0478, 0.0274, 0.0500], [[[-0.0478, 0.0274, 0.0500],
[-0.0394, 0.0032, -0.0496], [-0.0394, 0.0032, -0.0496],
[ 0.0381, 0.0391, 0.0330]], [ 0.0381, 0.0391, 0.0330]],
[[ 0.0184, -0.0560, -0.0345], [[ 0.0184, -0.0560, -0.0345],
[-0.0459, -0.0215, 0.0452], [-0.0459, -0.0215, 0.0452],
[ 0.0049, 0.0537, 0.0544]], [ 0.0049, 0.0537, 0.0544]],
[[-0.0413, -0.0084, 0.0585], [[-0.0413, -0.0084, 0.0585],
[ 0.0338, -0.0067, -0.0113], [ 0.0338, -0.0067, -0.0113],
[-0.0187, -0.0234, -0.0525]], [-0.0187, -0.0234, -0.0525]],
..., ...,
[[-0.0389, 0.0325, -0.0538], [[-0.0389, 0.0325, -0.0538],
[ 0.0118, 0.0509, 0.0352], [ 0.0118, 0.0509, 0.0352],
[-0.0351, -0.0341, -0.0506]], [-0.0351, -0.0341, -0.0506]],
[[ 0.0136, -0.0349, 0.0082], [[ 0.0136, -0.0349, 0.0082],
[ 0.0358, -0.0211, 0.0537], [ 0.0358, -0.0211, 0.0537],
[-0.0183, 0.0390, -0.0267]], [-0.0183, 0.0390, -0.0267]],
[[-0.0219, -0.0145, -0.0351], [[-0.0219, -0.0145, -0.0351],
[ 0.0556, 0.0033, -0.0030], [ 0.0556, 0.0033, -0.0030],
[ 0.0075, -0.0425, -0.0365]]]])), [ 0.0075, -0.0425, -0.0365]]]])),
('conv3.bias', ('conv3.bias',
tensor([ 0.0304, 0.0099, -0.0004, 0.0334, 0.0301, 0.0491, 0.0530, -0.0432, tensor([ 0.0304, 0.0099, -0.0004, 0.0334, 0.0301, 0.0491, 0.0530, -0.0432,
-0.0127, -0.0549, -0.0419, 0.0159, -0.0284, 0.0295, -0.0148, 0.0275, -0.0127, -0.0549, -0.0419, 0.0159, -0.0284, 0.0295, -0.0148, 0.0275,
0.0554, -0.0056, 0.0389, -0.0264, -0.0383, 0.0126, 0.0320, 0.0312, 0.0554, -0.0056, 0.0389, -0.0264, -0.0383, 0.0126, 0.0320, 0.0312,
0.0018, 0.0560, -0.0329, -0.0155, -0.0391, -0.0539, -0.0571, -0.0254])), 0.0018, 0.0560, -0.0329, -0.0155, -0.0391, -0.0539, -0.0571, -0.0254])),
('conv4.weight', ('conv4.weight',
tensor([[[[-3.8911e-02, -3.4220e-02, 4.2567e-03], tensor([[[[-3.8911e-02, -3.4220e-02, 4.2567e-03],
[-4.5321e-02, -5.2531e-02, -8.1722e-03], [-4.5321e-02, -5.2531e-02, -8.1722e-03],
[-2.2638e-02, 4.4213e-02, 5.6989e-02]], [-2.2638e-02, 4.4213e-02, 5.6989e-02]],
[[ 1.8417e-03, -1.4453e-02, 4.9892e-02], [[ 1.8417e-03, -1.4453e-02, 4.9892e-02],
[ 5.7762e-02, 9.6610e-03, -3.9509e-02], [ 5.7762e-02, 9.6610e-03, -3.9509e-02],
[ 3.3795e-02, 5.0409e-02, -5.8834e-02]], [ 3.3795e-02, 5.0409e-02, -5.8834e-02]],
[[ 4.6645e-03, -1.6286e-02, 4.3410e-02], [[ 4.6645e-03, -1.6286e-02, 4.3410e-02],
[-3.4043e-02, -2.2207e-02, 4.0967e-02], [-3.4043e-02, -2.2207e-02, 4.0967e-02],
[ 5.3004e-02, -2.2756e-02, -6.7993e-03]], [ 5.3004e-02, -2.2756e-02, -6.7993e-03]],
..., ...,
[[ 1.1741e-02, -5.5062e-02, -3.3625e-02], [[ 1.1741e-02, -5.5062e-02, -3.3625e-02],
[-9.2320e-03, -3.3036e-02, 3.3196e-02], [-9.2320e-03, -3.3036e-02, 3.3196e-02],
[ 2.3940e-02, 2.0442e-02, 1.4183e-02]], [ 2.3940e-02, 2.0442e-02, 1.4183e-02]],
[[-2.7139e-02, -3.4129e-03, -1.0090e-02], [[-2.7139e-02, -3.4129e-03, -1.0090e-02],
[ 1.3073e-02, -1.6998e-02, -4.7540e-02], [ 1.3073e-02, -1.6998e-02, -4.7540e-02],
[-2.5758e-02, -1.9363e-02, 2.1905e-02]], [-2.5758e-02, -1.9363e-02, 2.1905e-02]],
[[ 5.7593e-02, 1.5013e-02, -5.7894e-02], [[ 5.7593e-02, 1.5013e-02, -5.7894e-02],
[ 5.7964e-02, -2.3412e-02, 2.6955e-02], [ 5.7964e-02, -2.3412e-02, 2.6955e-02],
[-3.9814e-02, -4.6015e-02, -5.3240e-02]]], [-3.9814e-02, -4.6015e-02, -5.3240e-02]]],
[[[ 5.8211e-02, -4.1118e-02, 2.7704e-02], [[[ 5.8211e-02, -4.1118e-02, 2.7704e-02],
[ 5.7198e-02, 8.4165e-03, -5.1708e-02], [ 5.7198e-02, 8.4165e-03, -5.1708e-02],
[ 3.1423e-02, 1.5026e-02, 3.5922e-02]], [ 3.1423e-02, 1.5026e-02, 3.5922e-02]],
[[ 8.8858e-03, 3.2818e-02, 5.4486e-02], [[ 8.8858e-03, 3.2818e-02, 5.4486e-02],
[-2.6636e-02, 2.2604e-02, 2.9531e-02], [-2.6636e-02, 2.2604e-02, 2.9531e-02],
[-1.0327e-03, 2.2348e-03, 2.4103e-02]], [-1.0327e-03, 2.2348e-03, 2.4103e-02]],
[[ 3.8683e-02, -5.0057e-03, 5.0224e-02], [[ 3.8683e-02, -5.0057e-03, 5.0224e-02],
[ 3.5756e-02, -2.7295e-02, -2.2854e-02], [ 3.5756e-02, -2.7295e-02, -2.2854e-02],
[-3.2043e-02, -3.2415e-02, 4.1034e-02]], [-3.2043e-02, -3.2415e-02, 4.1034e-02]],
..., ...,
[[-1.9791e-02, 4.3243e-02, -3.5177e-02], [[-1.9791e-02, 4.3243e-02, -3.5177e-02],
[ 2.4554e-02, 4.2845e-03, 4.8009e-02], [ 2.4554e-02, 4.2845e-03, 4.8009e-02],
[ 2.4897e-03, 3.9550e-02, -3.0833e-02]], [ 2.4897e-03, 3.9550e-02, -3.0833e-02]],
[[ 4.5807e-02, 6.5845e-03, 9.3362e-05], [[ 4.5807e-02, 6.5845e-03, 9.3362e-05],
[-1.9411e-02, -2.9161e-02, 5.0828e-02], [-1.9411e-02, -2.9161e-02, 5.0828e-02],
[ 1.2028e-03, 2.1260e-02, -4.3710e-03]], [ 1.2028e-03, 2.1260e-02, -4.3710e-03]],
[[-4.8702e-02, -2.0571e-02, -3.5162e-02], [[-4.8702e-02, -2.0571e-02, -3.5162e-02],
[-2.5856e-02, -2.6619e-02, 8.1867e-03], [-2.5856e-02, -2.6619e-02, 8.1867e-03],
[-2.7671e-02, -9.6651e-03, -5.3279e-02]]], [-2.7671e-02, -9.6651e-03, -5.3279e-02]]],
[[[-5.6432e-02, 2.3722e-02, -2.1750e-02], [[[-5.6432e-02, 2.3722e-02, -2.1750e-02],
[-4.8247e-03, -2.1226e-02, -1.0829e-02], [-4.8247e-03, -2.1226e-02, -1.0829e-02],
[-1.9523e-02, -1.8187e-02, 2.2772e-03]], [-1.9523e-02, -1.8187e-02, 2.2772e-03]],
[[-1.0907e-02, 3.3984e-02, -1.2088e-02], [[-1.0907e-02, 3.3984e-02, -1.2088e-02],
[-1.8657e-02, -4.8297e-02, 2.3614e-02], [-1.8657e-02, -4.8297e-02, 2.3614e-02],
[-3.9670e-02, 6.1733e-03, -2.9168e-02]], [-3.9670e-02, 6.1733e-03, -2.9168e-02]],
[[-4.2112e-02, 2.8203e-02, -1.7385e-03], [[-4.2112e-02, 2.8203e-02, -1.7385e-03],
[-2.5282e-02, -9.4592e-05, 6.5093e-03], [-2.5282e-02, -9.4592e-05, 6.5093e-03],
[-4.1745e-02, 4.3988e-03, -1.1622e-02]], [-4.1745e-02, 4.3988e-03, -1.1622e-02]],
..., ...,
[[ 4.2991e-02, 1.6912e-02, -4.3689e-02], [[ 4.2991e-02, 1.6912e-02, -4.3689e-02],
[ 5.1871e-02, 4.8566e-02, 3.6205e-02], [ 5.1871e-02, 4.8566e-02, 3.6205e-02],
[-3.2016e-02, -1.3596e-02, -2.7950e-02]], [-3.2016e-02, -1.3596e-02, -2.7950e-02]],
[[-2.8307e-02, -4.0278e-02, 1.5087e-02], [[-2.8307e-02, -4.0278e-02, 1.5087e-02],
[ 4.0443e-02, -3.5727e-02, 3.7196e-02], [ 4.0443e-02, -3.5727e-02, 3.7196e-02],
[-1.4194e-02, -2.7319e-02, -5.1305e-02]], [-1.4194e-02, -2.7319e-02, -5.1305e-02]],
[[-2.9962e-02, 2.4693e-02, -4.4912e-02], [[-2.9962e-02, 2.4693e-02, -4.4912e-02],
[ 5.5890e-03, 4.6671e-02, 3.3599e-02], [ 5.5890e-03, 4.6671e-02, 3.3599e-02],
[-3.9949e-02, -4.4716e-02, 2.2345e-02]]], [-3.9949e-02, -4.4716e-02, 2.2345e-02]]],
..., ...,
[[[ 3.1920e-02, -4.9932e-02, -1.0871e-02], [[[ 3.1920e-02, -4.9932e-02, -1.0871e-02],
[-3.7500e-02, 4.1638e-02, -1.3246e-02], [-3.7500e-02, 4.1638e-02, -1.3246e-02],
[ 1.6447e-02, -5.6741e-02, -3.7524e-02]], [ 1.6447e-02, -5.6741e-02, -3.7524e-02]],
[[ 3.3903e-02, -3.1321e-02, -4.4877e-02], [[ 3.3903e-02, -3.1321e-02, -4.4877e-02],
[-2.2473e-02, -2.4225e-02, 4.5838e-02], [-2.2473e-02, -2.4225e-02, 4.5838e-02],
[-2.0069e-02, 3.8338e-02, 5.8010e-02]], [-2.0069e-02, 3.8338e-02, 5.8010e-02]],
[[ 1.7602e-02, -5.2530e-02, 4.9331e-02], [[ 1.7602e-02, -5.2530e-02, 4.9331e-02],
[ 2.4509e-02, 2.3943e-02, -2.1774e-02], [ 2.4509e-02, 2.3943e-02, -2.1774e-02],
[-5.7154e-02, 5.7090e-02, 3.7531e-02]], [-5.7154e-02, 5.7090e-02, 3.7531e-02]],
..., ...,
[[ 2.8630e-02, -4.8644e-04, -5.3822e-02], [[ 2.8630e-02, -4.8644e-04, -5.3822e-02],
[-1.1102e-02, 4.8524e-02, -2.7142e-02], [-1.1102e-02, 4.8524e-02, -2.7142e-02],
[-5.3463e-02, 3.5607e-02, -1.1110e-02]], [-5.3463e-02, 3.5607e-02, -1.1110e-02]],
[[ 4.7891e-02, -3.4098e-02, 3.6984e-02], [[ 4.7891e-02, -3.4098e-02, 3.6984e-02],
[ 3.9062e-02, -5.1119e-03, -3.3252e-02], [ 3.9062e-02, -5.1119e-03, -3.3252e-02],
[ 5.5029e-02, 3.1092e-03, -5.9391e-03]], [ 5.5029e-02, 3.1092e-03, -5.9391e-03]],
[[ 9.6775e-03, 2.2903e-02, -1.5971e-02], [[ 9.6775e-03, 2.2903e-02, -1.5971e-02],
[-2.6969e-02, 8.5069e-04, -2.5744e-02], [-2.6969e-02, 8.5069e-04, -2.5744e-02],
[ 2.7311e-03, -2.2119e-02, 2.4367e-03]]], [ 2.7311e-03, -2.2119e-02, 2.4367e-03]]],
[[[ 1.5206e-02, -5.3504e-02, -1.8013e-02], [[[ 1.5206e-02, -5.3504e-02, -1.8013e-02],
[-3.0064e-02, 3.2887e-02, 1.7612e-02], [-3.0064e-02, 3.2887e-02, 1.7612e-02],
[-4.2775e-02, -2.7335e-02, 5.1532e-02]], [-4.2775e-02, -2.7335e-02, 5.1532e-02]],
[[ 4.3325e-02, 1.2909e-03, 5.6831e-02], [[ 4.3325e-02, 1.2909e-03, 5.6831e-02],
[ 4.8283e-03, -4.4274e-02, 1.7624e-02], [ 4.8283e-03, -4.4274e-02, 1.7624e-02],
[-7.2574e-03, 1.3743e-02, -5.1502e-02]], [-7.2574e-03, 1.3743e-02, -5.1502e-02]],
[[-1.4299e-02, -3.0024e-02, 4.9578e-02], [[-1.4299e-02, -3.0024e-02, 4.9578e-02],
[-4.6143e-02, -3.0686e-02, -5.0727e-02], [-4.6143e-02, -3.0686e-02, -5.0727e-02],
[ 5.4766e-02, -1.4012e-02, 3.3267e-02]], [ 5.4766e-02, -1.4012e-02, 3.3267e-02]],
..., ...,
[[ 5.2773e-02, 4.0294e-02, -2.6113e-02], [[ 5.2773e-02, 4.0294e-02, -2.6113e-02],
[-2.5069e-02, 4.3956e-02, -4.7841e-02], [-2.5069e-02, 4.3956e-02, -4.7841e-02],
[ 3.0924e-02, 1.5174e-02, -4.8323e-02]], [ 3.0924e-02, 1.5174e-02, -4.8323e-02]],
[[-2.0325e-02, 3.2666e-02, 2.5174e-02], [[-2.0325e-02, 3.2666e-02, 2.5174e-02],
[ 5.3775e-03, -3.2712e-02, -5.2251e-02], [ 5.3775e-03, -3.2712e-02, -5.2251e-02],
[-2.7426e-02, -4.5502e-04, -3.2174e-02]], [-2.7426e-02, -4.5502e-04, -3.2174e-02]],
[[ 8.5780e-03, -5.2099e-02, 5.7285e-02], [[ 8.5780e-03, -5.2099e-02, 5.7285e-02],
[-5.0897e-02, -5.3995e-03, 4.2719e-02], [-5.0897e-02, -5.3995e-03, 4.2719e-02],
[-5.2257e-02, -7.9682e-03, 2.1848e-02]]], [-5.2257e-02, -7.9682e-03, 2.1848e-02]]],
[[[-2.8660e-02, 1.2392e-02, 2.9940e-02], [[[-2.8660e-02, 1.2392e-02, 2.9940e-02],
[-1.1170e-02, -1.7499e-02, -5.3951e-02], [-1.1170e-02, -1.7499e-02, -5.3951e-02],
[ 4.9738e-02, 2.7240e-02, 2.9588e-03]], [ 4.9738e-02, 2.7240e-02, 2.9588e-03]],
[[ 3.7728e-02, -3.1084e-02, 4.6459e-02], [[ 3.7728e-02, -3.1084e-02, 4.6459e-02],
[-1.4961e-02, 1.6951e-02, -1.3976e-02], [-1.4961e-02, 1.6951e-02, -1.3976e-02],
[ 2.5768e-02, -3.8991e-02, -2.8738e-02]], [ 2.5768e-02, -3.8991e-02, -2.8738e-02]],
[[ 1.6084e-02, -7.3174e-03, -4.5839e-02], [[ 1.6084e-02, -7.3174e-03, -4.5839e-02],
[-3.6029e-02, 1.5303e-02, -5.4380e-02], [-3.6029e-02, 1.5303e-02, -5.4380e-02],
[ 2.1913e-02, -4.4792e-02, 4.7973e-02]], [ 2.1913e-02, -4.4792e-02, 4.7973e-02]],
..., ...,
[[ 5.4945e-02, -4.5541e-02, -1.5806e-02], [[ 5.4945e-02, -4.5541e-02, -1.5806e-02],
[-4.5216e-02, -2.0338e-02, 3.3373e-02], [-4.5216e-02, -2.0338e-02, 3.3373e-02],
[-1.8431e-02, 4.3953e-02, -4.8196e-02]], [-1.8431e-02, 4.3953e-02, -4.8196e-02]],
[[-3.6129e-02, 5.7705e-02, -1.2229e-02], [[-3.6129e-02, 5.7705e-02, -1.2229e-02],
[-5.2801e-02, 7.0930e-03, -2.6721e-02], [-5.2801e-02, 7.0930e-03, -2.6721e-02],
[ 3.6720e-02, 7.5540e-04, 5.5401e-02]], [ 3.6720e-02, 7.5540e-04, 5.5401e-02]],
[[-4.0386e-02, 1.6714e-02, 2.9246e-02], [[-4.0386e-02, 1.6714e-02, 2.9246e-02],
[ 4.7033e-02, -8.7923e-03, 5.1267e-02], [ 4.7033e-02, -8.7923e-03, 5.1267e-02],
[-3.1211e-02, -2.7036e-02, 4.0346e-02]]]])), [-3.1211e-02, -2.7036e-02, 4.0346e-02]]]])),
('conv4.bias', ('conv4.bias',
tensor([-0.0368, -0.0535, -0.0568, -0.0364, 0.0167, 0.0255, 0.0031, 0.0562, tensor([-0.0368, -0.0535, -0.0568, -0.0364, 0.0167, 0.0255, 0.0031, 0.0562,
-0.0575, 0.0470, -0.0128, 0.0159, -0.0387, 0.0517, -0.0508, -0.0104, -0.0575, 0.0470, -0.0128, 0.0159, -0.0387, 0.0517, -0.0508, -0.0104,
0.0133, -0.0265, 0.0290, 0.0147, -0.0405, 0.0074, 0.0434, -0.0436, 0.0133, -0.0265, 0.0290, 0.0147, -0.0405, 0.0074, 0.0434, -0.0436,
-0.0295, 0.0417, 0.0324, 0.0316, 0.0143, -0.0312, 0.0463, 0.0441])), -0.0295, 0.0417, 0.0324, 0.0316, 0.0143, -0.0312, 0.0463, 0.0441])),
('fc1.weight', ('fc1.weight',
tensor([[-0.0206, -0.0161, -0.0053, ..., -0.0257, -0.0314, -0.0105], tensor([[-0.0206, -0.0161, -0.0053, ..., -0.0257, -0.0314, -0.0105],
[-0.0229, 0.0030, 0.0239, ..., 0.0199, -0.0279, 0.0155]])), [-0.0229, 0.0030, 0.0239, ..., 0.0199, -0.0279, 0.0155]])),
('fc1.bias', tensor([-0.0333, -0.0126]))]) ('fc1.bias', tensor([-0.0333, -0.0126]))])
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
m2 = m1.state_dict() m2 = m1.state_dict()
m2['fc1.bias'] = torch.Tensor([-0.5, -1.2]) m2['fc1.bias'] = torch.Tensor([-0.5, -1.2])
m1.load_state_dict(m2) m1.load_state_dict(m2)
m1.state_dict() m1.state_dict()
``` ```
%% Output %% Output
OrderedDict([('conv1.weight', OrderedDict([('conv1.weight',
tensor([[[[-0.1261, 0.1833, -0.1406], tensor([[[[-0.1261, 0.1833, -0.1406],
[ 0.1324, -0.0685, 0.0938], [ 0.1324, -0.0685, 0.0938],
[ 0.0432, 0.1814, -0.0541]], [ 0.0432, 0.1814, -0.0541]],
[[-0.1776, -0.1839, -0.0111], [[-0.1776, -0.1839, -0.0111],
[ 0.0888, 0.0888, -0.1344], [ 0.0888, 0.0888, -0.1344],
[-0.1838, 0.1737, 0.1584]], [-0.1838, 0.1737, 0.1584]],
[[ 0.0417, 0.1064, -0.0156], [[ 0.0417, 0.1064, -0.0156],
[ 0.0667, 0.0856, -0.1746], [ 0.0667, 0.0856, -0.1746],
[ 0.0412, 0.1620, 0.0125]]], [ 0.0412, 0.1620, 0.0125]]],
[[[-0.0530, -0.1273, -0.0797], [[[-0.0530, -0.1273, -0.0797],
[ 0.0422, 0.1135, 0.0475], [ 0.0422, 0.1135, 0.0475],
[-0.0244, 0.1691, -0.1383]], [-0.0244, 0.1691, -0.1383]],
[[ 0.0822, -0.1317, -0.1692], [[ 0.0822, -0.1317, -0.1692],
[ 0.1373, 0.1388, 0.0103], [ 0.1373, 0.1388, 0.0103],
[-0.0481, 0.1105, 0.0631]], [-0.0481, 0.1105, 0.0631]],
[[-0.0352, 0.1259, -0.0530], [[-0.0352, 0.1259, -0.0530],
[-0.1394, -0.0281, 0.1844], [-0.1394, -0.0281, 0.1844],
[ 0.0082, 0.1187, 0.0211]]], [ 0.0082, 0.1187, 0.0211]]],
[[[ 0.0987, 0.0788, -0.1126], [[[ 0.0987, 0.0788, -0.1126],
[ 0.1769, 0.0763, -0.1767], [ 0.1769, 0.0763, -0.1767],
[-0.0570, 0.1156, 0.1770]], [-0.0570, 0.1156, 0.1770]],
[[ 0.0643, -0.0024, -0.0625], [[ 0.0643, -0.0024, -0.0625],
[ 0.0819, 0.0140, -0.1882], [ 0.0819, 0.0140, -0.1882],
[ 0.1325, -0.0632, -0.0202]], [ 0.1325, -0.0632, -0.0202]],
[[ 0.0053, 0.1042, -0.0058], [[ 0.0053, 0.1042, -0.0058],
[-0.1082, -0.1753, 0.1762], [-0.1082, -0.1753, 0.1762],
[-0.0501, 0.1166, 0.0561]]], [-0.0501, 0.1166, 0.0561]]],
[[[ 0.0358, -0.0685, -0.1278], [[[ 0.0358, -0.0685, -0.1278],
[ 0.0029, -0.1107, 0.1169], [ 0.0029, -0.1107, 0.1169],
[-0.1408, 0.1293, 0.1142]], [-0.1408, 0.1293, 0.1142]],
[[-0.0814, 0.0470, 0.0188], [[-0.0814, 0.0470, 0.0188],
[ 0.1538, 0.0137, 0.1128], [ 0.1538, 0.0137, 0.1128],
[-0.1597, 0.1432, 0.1370]], [-0.1597, 0.1432, 0.1370]],
[[ 0.1425, 0.1769, -0.0037], [[ 0.1425, 0.1769, -0.0037],
[-0.1080, -0.0805, -0.0195], [-0.1080, -0.0805, -0.0195],
[-0.1335, -0.1666, 0.1399]]], [-0.1335, -0.1666, 0.1399]]],
[[[ 0.1117, 0.1918, -0.1666], [[[ 0.1117, 0.1918, -0.1666],
[-0.1392, 0.0086, 0.0172], [-0.1392, 0.0086, 0.0172],
[-0.0721, -0.1711, 0.0344]], [-0.0721, -0.1711, 0.0344]],
[[ 0.1820, -0.0537, -0.0974], [[ 0.1820, -0.0537, -0.0974],
[ 0.0366, -0.0710, 0.1273], [ 0.0366, -0.0710, 0.1273],
[ 0.1132, -0.1594, 0.0878]], [ 0.1132, -0.1594, 0.0878]],
[[-0.0874, -0.0401, 0.1827], [[-0.0874, -0.0401, 0.1827],
[-0.0301, 0.1205, -0.0396], [-0.0301, 0.1205, -0.0396],
[-0.1143, -0.1007, 0.1561]]], [-0.1143, -0.1007, 0.1561]]],
[[[ 0.1522, -0.0012, -0.1785], [[[ 0.1522, -0.0012, -0.1785],
[-0.1833, -0.1828, -0.1643], [-0.1833, -0.1828, -0.1643],
[-0.1765, -0.1757, -0.0608]], [-0.1765, -0.1757, -0.0608]],
[[-0.0684, 0.0521, 0.1137], [[-0.0684, 0.0521, 0.1137],
[-0.0028, 0.0616, 0.0758], [-0.0028, 0.0616, 0.0758],
[-0.1736, 0.0667, 0.1229]], [-0.1736, 0.0667, 0.1229]],
[[ 0.1298, -0.1848, -0.1570], [[ 0.1298, -0.1848, -0.1570],
[-0.1052, -0.1172, -0.1223], [-0.1052, -0.1172, -0.1223],
[-0.1389, -0.0095, -0.0410]]], [-0.1389, -0.0095, -0.0410]]],
[[[ 0.0213, -0.0975, 0.0964], [[[ 0.0213, -0.0975, 0.0964],
[ 0.0535, -0.0775, 0.0790], [ 0.0535, -0.0775, 0.0790],
[-0.1796, -0.1468, 0.1036]], [-0.1796, -0.1468, 0.1036]],
[[-0.0403, 0.0646, -0.0932], [[-0.0403, 0.0646, -0.0932],
[ 0.1779, -0.1616, 0.0644], [ 0.1779, -0.1616, 0.0644],
[-0.0508, -0.1158, -0.0592]], [-0.0508, -0.1158, -0.0592]],
[[-0.1644, -0.1327, 0.0817], [[-0.1644, -0.1327, 0.0817],
[ 0.0320, -0.0213, -0.0946], [ 0.0320, -0.0213, -0.0946],
[-0.1106, 0.1463, -0.1642]]], [-0.1106, 0.1463, -0.1642]]],
[[[-0.0985, -0.1160, -0.0915], [[[-0.0985, -0.1160, -0.0915],
[ 0.1857, 0.0806, 0.1761], [ 0.1857, 0.0806, 0.1761],
[-0.0817, 0.1095, 0.0896]], [-0.0817, 0.1095, 0.0896]],
[[-0.0660, -0.1680, 0.1833], [[-0.0660, -0.1680, 0.1833],
[ 0.0611, 0.0077, -0.0848], [ 0.0611, 0.0077, -0.0848],
[-0.1516, 0.1737, 0.0484]], [-0.1516, 0.1737, 0.0484]],
[[ 0.1434, -0.0732, -0.0904], [[ 0.1434, -0.0732, -0.0904],
[ 0.0962, 0.1783, 0.0192], [ 0.0962, 0.1783, 0.0192],
[ 0.0915, 0.0006, 0.0334]]], [ 0.0915, 0.0006, 0.0334]]],
[[[-0.0047, 0.1807, -0.1798], [[[-0.0047, 0.1807, -0.1798],
[-0.0164, 0.1119, -0.0805], [-0.0164, 0.1119, -0.0805],
[ 0.1855, -0.0681, -0.0187]], [ 0.1855, -0.0681, -0.0187]],
[[-0.0069, 0.0491, -0.1868], [[-0.0069, 0.0491, -0.1868],
[-0.1609, -0.0316, 0.0150], [-0.1609, -0.0316, 0.0150],
[-0.1605, 0.1506, -0.0074]], [-0.1605, 0.1506, -0.0074]],
[[ 0.0851, -0.1732, -0.1777], [[ 0.0851, -0.1732, -0.1777],
[ 0.0539, -0.0500, -0.1231], [ 0.0539, -0.0500, -0.1231],
[ 0.1654, 0.0342, -0.1904]]], [ 0.1654, 0.0342, -0.1904]]],
[[[ 0.0476, 0.0284, 0.1212], [[[ 0.0476, 0.0284, 0.1212],
[-0.1603, -0.1924, 0.0144], [-0.1603, -0.1924, 0.0144],
[ 0.0076, -0.0928, -0.1645]], [ 0.0076, -0.0928, -0.1645]],
[[ 0.0215, 0.1845, -0.1034], [[ 0.0215, 0.1845, -0.1034],
[ 0.1574, -0.1577, -0.0438], [ 0.1574, -0.1577, -0.0438],
[-0.1360, -0.0601, -0.1693]], [-0.1360, -0.0601, -0.1693]],
[[-0.0720, 0.0619, 0.1405], [[-0.0720, 0.0619, 0.1405],
[ 0.0699, -0.1288, 0.0041], [ 0.0699, -0.1288, 0.0041],
[-0.0381, -0.1697, -0.1568]]], [-0.0381, -0.1697, -0.1568]]],
[[[-0.1599, 0.1231, -0.1034], [[[-0.1599, 0.1231, -0.1034],
[-0.0314, 0.0105, -0.1449], [-0.0314, 0.0105, -0.1449],
[-0.0172, -0.0781, 0.0839]], [-0.0172, -0.0781, 0.0839]],
[[-0.0676, 0.1185, -0.1559], [[-0.0676, 0.1185, -0.1559],
[-0.1053, -0.1306, 0.1820], [-0.1053, -0.1306, 0.1820],
[ 0.1584, -0.1370, 0.1828]], [ 0.1584, -0.1370, 0.1828]],
[[ 0.0658, 0.1412, -0.0537], [[ 0.0658, 0.1412, -0.0537],
[-0.1230, -0.1411, -0.0011], [-0.1230, -0.1411, -0.0011],
[-0.1318, -0.0458, 0.1838]]], [-0.1318, -0.0458, 0.1838]]],
[[[-0.0268, 0.1747, -0.1037], [[[-0.0268, 0.1747, -0.1037],
[ 0.0515, -0.0228, -0.1024], [ 0.0515, -0.0228, -0.1024],
[-0.1543, -0.0643, -0.0100]], [-0.1543, -0.0643, -0.0100]],
[[-0.1572, -0.1530, 0.0026], [[-0.1572, -0.1530, 0.0026],
[ 0.1463, -0.1233, 0.0470], [ 0.1463, -0.1233, 0.0470],
[-0.1595, -0.1108, -0.0654]], [-0.1595, -0.1108, -0.0654]],
[[-0.0521, -0.0094, 0.1544], [[-0.0521, -0.0094, 0.1544],
[-0.0505, -0.0332, 0.0048], [-0.0505, -0.0332, 0.0048],
[ 0.0735, 0.1350, 0.0690]]], [ 0.0735, 0.1350, 0.0690]]],
[[[ 0.0025, 0.0724, 0.0930], [[[ 0.0025, 0.0724, 0.0930],
[-0.1885, 0.0475, 0.1100], [-0.1885, 0.0475, 0.1100],
[-0.1622, 0.0087, -0.0030]], [-0.1622, 0.0087, -0.0030]],
[[ 0.1032, -0.1425, -0.0620], [[ 0.1032, -0.1425, -0.0620],
[ 0.1515, -0.0736, -0.1888], [ 0.1515, -0.0736, -0.1888],
[-0.1246, 0.1424, -0.0491]], [-0.1246, 0.1424, -0.0491]],
[[ 0.1759, -0.1616, 0.1198], [[ 0.1759, -0.1616, 0.1198],
[-0.1103, 0.1032, 0.1727], [-0.1103, 0.1032, 0.1727],
[-0.0601, 0.1635, 0.0034]]], [-0.0601, 0.1635, 0.0034]]],
[[[ 0.0301, 0.1517, 0.0657], [[[ 0.0301, 0.1517, 0.0657],
[-0.1368, -0.1165, 0.1193], [-0.1368, -0.1165, 0.1193],
[-0.0962, 0.1451, 0.1099]], [-0.0962, 0.1451, 0.1099]],
[[ 0.1646, -0.1860, -0.1187], [[ 0.1646, -0.1860, -0.1187],
[-0.1367, -0.0911, 0.1337], [-0.1367, -0.0911, 0.1337],
[-0.0926, -0.0524, -0.0672]], [-0.0926, -0.0524, -0.0672]],
[[-0.1509, -0.1231, -0.0855], [[-0.1509, -0.1231, -0.0855],
[ 0.1808, -0.0713, 0.0410], [ 0.1808, -0.0713, 0.0410],
[-0.0621, -0.0506, 0.1871]]], [-0.0621, -0.0506, 0.1871]]],
[[[ 0.0888, -0.0874, -0.0826], [[[ 0.0888, -0.0874, -0.0826],
[ 0.0416, -0.0961, 0.0603], [ 0.0416, -0.0961, 0.0603],
[ 0.1455, 0.0050, 0.0318]], [ 0.1455, 0.0050, 0.0318]],
[[-0.1633, 0.0070, -0.1537], [[-0.1633, 0.0070, -0.1537],
[-0.0109, 0.1602, -0.0463], [-0.0109, 0.1602, -0.0463],
[-0.0423, -0.0147, -0.1045]], [-0.0423, -0.0147, -0.1045]],
[[ 0.1640, -0.0997, -0.1662], [[ 0.1640, -0.0997, -0.1662],
[-0.1074, 0.1549, -0.1905], [-0.1074, 0.1549, -0.1905],
[-0.1708, 0.1624, 0.0219]]], [-0.1708, 0.1624, 0.0219]]],
[[[ 0.0824, -0.1376, 0.1086], [[[ 0.0824, -0.1376, 0.1086],
[ 0.0836, 0.0135, 0.0351], [ 0.0836, 0.0135, 0.0351],
[-0.1518, 0.0784, -0.1708]], [-0.1518, 0.0784, -0.1708]],
[[-0.1636, -0.1571, 0.1032], [[-0.1636, -0.1571, 0.1032],
[-0.1152, 0.0274, -0.1022], [-0.1152, 0.0274, -0.1022],
[-0.0956, -0.1606, -0.1615]], [-0.0956, -0.1606, -0.1615]],
[[ 0.1307, 0.0419, 0.1924], [[ 0.1307, 0.0419, 0.1924],
[-0.0599, -0.1296, -0.0448], [-0.0599, -0.1296, -0.0448],
[ 0.0363, 0.0377, -0.0460]]], [ 0.0363, 0.0377, -0.0460]]],
[[[-0.1685, -0.1277, -0.0465], [[[-0.1685, -0.1277, -0.0465],
[ 0.0922, -0.1011, 0.0742], [ 0.0922, -0.1011, 0.0742],
[ 0.0053, -0.1456, 0.0135]], [ 0.0053, -0.1456, 0.0135]],
[[ 0.1341, 0.0131, 0.1281], [[ 0.1341, 0.0131, 0.1281],
[-0.1020, 0.1069, -0.0631], [-0.1020, 0.1069, -0.0631],
[-0.0439, -0.1189, -0.1822]], [-0.0439, -0.1189, -0.1822]],
[[ 0.1624, -0.1253, 0.0302], [[ 0.1624, -0.1253, 0.0302],
[ 0.0709, 0.0767, 0.1453], [ 0.0709, 0.0767, 0.1453],
[ 0.0203, 0.1603, -0.1720]]], [ 0.0203, 0.1603, -0.1720]]],
[[[-0.1550, 0.1513, -0.1003], [[[-0.1550, 0.1513, -0.1003],
[ 0.0370, 0.0367, -0.0233], [ 0.0370, 0.0367, -0.0233],
[ 0.0916, -0.0871, 0.1579]], [ 0.0916, -0.0871, 0.1579]],
[[-0.1900, 0.0314, 0.0865], [[-0.1900, 0.0314, 0.0865],
[-0.0197, 0.0296, -0.0048], [-0.0197, 0.0296, -0.0048],
[ 0.0846, 0.1543, -0.0770]], [ 0.0846, 0.1543, -0.0770]],
[[-0.0016, -0.0978, 0.1826], [[-0.0016, -0.0978, 0.1826],
[-0.0477, 0.0689, 0.1079], [-0.0477, 0.0689, 0.1079],
[ 0.0400, 0.0880, 0.1674]]], [ 0.0400, 0.0880, 0.1674]]],
[[[ 0.0145, -0.0447, -0.1742], [[[ 0.0145, -0.0447, -0.1742],
[ 0.0394, 0.0127, -0.1172], [ 0.0394, 0.0127, -0.1172],
[ 0.1330, -0.1207, 0.0326]], [ 0.1330, -0.1207, 0.0326]],
[[-0.0155, -0.1602, 0.0023], [[-0.0155, -0.1602, 0.0023],
[ 0.0789, 0.1648, 0.1781], [ 0.0789, 0.1648, 0.1781],
[-0.1468, -0.0481, -0.1260]], [-0.1468, -0.0481, -0.1260]],
[[-0.0139, 0.0848, -0.0536], [[-0.0139, 0.0848, -0.0536],
[-0.1581, 0.1130, 0.0717], [-0.1581, 0.1130, 0.0717],
[ 0.0275, -0.0006, -0.0049]]], [ 0.0275, -0.0006, -0.0049]]],
[[[-0.0199, 0.0032, -0.1246], [[[-0.0199, 0.0032, -0.1246],
[ 0.0479, 0.1418, -0.1295], [ 0.0479, 0.1418, -0.1295],
[-0.1646, -0.1139, -0.1018]], [-0.1646, -0.1139, -0.1018]],
[[ 0.1475, 0.1413, -0.0354], [[ 0.1475, 0.1413, -0.0354],
[ 0.0612, -0.1652, 0.0801], [ 0.0612, -0.1652, 0.0801],
[-0.1306, -0.0165, 0.1733]], [-0.1306, -0.0165, 0.1733]],
[[ 0.1527, 0.0911, -0.1906], [[ 0.1527, 0.0911, -0.1906],
[-0.1152, 0.1737, 0.0436], [-0.1152, 0.1737, 0.0436],
[-0.0213, -0.0314, -0.0319]]], [-0.0213, -0.0314, -0.0319]]],
[[[-0.0003, -0.0546, -0.1255], [[[-0.0003, -0.0546, -0.1255],
[ 0.0914, -0.1414, 0.0542], [ 0.0914, -0.1414, 0.0542],
[ 0.1139, 0.0132, 0.0815]], [ 0.1139, 0.0132, 0.0815]],
[[-0.0042, 0.0541, 0.1456], [[-0.0042, 0.0541, 0.1456],
[ 0.0509, -0.0790, 0.0272], [ 0.0509, -0.0790, 0.0272],
[ 0.1419, 0.0992, -0.1448]], [ 0.1419, 0.0992, -0.1448]],
[[ 0.0496, 0.0013, 0.0838], [[ 0.0496, 0.0013, 0.0838],
[-0.0662, 0.0315, -0.1168], [-0.0662, 0.0315, -0.1168],
[-0.0069, -0.1503, 0.0729]]], [-0.0069, -0.1503, 0.0729]]],
[[[ 0.1866, 0.1329, -0.0560], [[[ 0.1866, 0.1329, -0.0560],
[ 0.0026, 0.1533, 0.0326], [ 0.0026, 0.1533, 0.0326],
[-0.1161, -0.0323, 0.0053]], [-0.1161, -0.0323, 0.0053]],
[[-0.0243, -0.1823, -0.1657], [[-0.0243, -0.1823, -0.1657],
[-0.0107, -0.0832, 0.0029], [-0.0107, -0.0832, 0.0029],
[ 0.0981, 0.1241, -0.1788]], [ 0.0981, 0.1241, -0.1788]],
[[-0.0400, -0.0577, -0.0757], [[-0.0400, -0.0577, -0.0757],
[-0.0584, 0.0176, -0.1019], [-0.0584, 0.0176, -0.1019],
[-0.1828, 0.1589, -0.0312]]], [-0.1828, 0.1589, -0.0312]]],
[[[-0.1083, -0.1236, -0.0904], [[[-0.1083, -0.1236, -0.0904],
[-0.1575, 0.0157, 0.0552], [-0.1575, 0.0157, 0.0552],
[-0.0839, 0.1704, -0.1457]], [-0.0839, 0.1704, -0.1457]],
[[-0.1648, -0.0270, -0.0489], [[-0.1648, -0.0270, -0.0489],
[-0.1122, -0.0288, -0.0073], [-0.1122, -0.0288, -0.0073],
[-0.1443, -0.1712, 0.0100]], [-0.1443, -0.1712, 0.0100]],
[[-0.1142, -0.1552, 0.1568], [[-0.1142, -0.1552, 0.1568],
[ 0.0743, -0.1108, -0.0643], [ 0.0743, -0.1108, -0.0643],
[-0.0394, -0.1345, 0.0992]]], [-0.0394, -0.1345, 0.0992]]],
[[[-0.1591, 0.0942, -0.1035], [[[-0.1591, 0.0942, -0.1035],
[-0.0781, 0.0725, -0.0888], [-0.0781, 0.0725, -0.0888],
[ 0.0959, 0.0213, 0.1222]], [ 0.0959, 0.0213, 0.1222]],
[[ 0.1202, -0.0217, -0.0955], [[ 0.1202, -0.0217, -0.0955],
[-0.1748, -0.1133, -0.0704], [-0.1748, -0.1133, -0.0704],
[-0.0670, -0.1401, 0.1553]], [-0.0670, -0.1401, 0.1553]],
[[ 0.0053, -0.0871, -0.0239], [[ 0.0053, -0.0871, -0.0239],
[ 0.0961, -0.0547, 0.1741], [ 0.0961, -0.0547, 0.1741],
[-0.0570, 0.0477, 0.1853]]], [-0.0570, 0.0477, 0.1853]]],
[[[-0.1115, -0.0183, -0.1302], [[[-0.1115, -0.0183, -0.1302],
[ 0.1435, -0.0238, -0.0048], [ 0.1435, -0.0238, -0.0048],
[ 0.1862, -0.1837, 0.1711]], [ 0.1862, -0.1837, 0.1711]],
[[ 0.1375, -0.1798, 0.0818], [[ 0.1375, -0.1798, 0.0818],
[-0.0792, 0.0820, 0.1373], [-0.0792, 0.0820, 0.1373],
[ 0.1849, 0.0672, -0.1822]], [ 0.1849, 0.0672, -0.1822]],
[[ 0.1868, -0.0356, 0.0726], [[ 0.1868, -0.0356, 0.0726],
[-0.1523, -0.1130, 0.1506], [-0.1523, -0.1130, 0.1506],
[-0.1046, 0.0178, 0.0990]]], [-0.1046, 0.0178, 0.0990]]],
[[[ 0.1321, -0.1641, 0.0411], [[[ 0.1321, -0.1641, 0.0411],
[ 0.0526, 0.0393, 0.0918], [ 0.0526, 0.0393, 0.0918],
[-0.1345, 0.0750, 0.0859]], [-0.1345, 0.0750, 0.0859]],
[[-0.0985, 0.1466, 0.1349], [[-0.0985, 0.1466, 0.1349],
[-0.1461, -0.1742, 0.0941], [-0.1461, -0.1742, 0.0941],
[-0.1502, -0.1813, 0.0864]], [-0.1502, -0.1813, 0.0864]],
[[-0.1039, 0.1179, 0.1499], [[-0.1039, 0.1179, 0.1499],
[-0.0366, -0.0120, 0.0951], [-0.0366, -0.0120, 0.0951],
[ 0.0087, 0.1212, -0.0183]]], [ 0.0087, 0.1212, -0.0183]]],
[[[-0.1375, 0.0765, -0.0072], [[[-0.1375, 0.0765, -0.0072],
[-0.0041, 0.0379, -0.0243], [-0.0041, 0.0379, -0.0243],
[-0.1495, 0.1601, 0.1575]], [-0.1495, 0.1601, 0.1575]],
[[-0.0454, 0.1642, 0.0720], [[-0.0454, 0.1642, 0.0720],
[-0.0533, 0.0150, 0.0039], [-0.0533, 0.0150, 0.0039],
[ 0.0194, 0.0113, -0.1194]], [ 0.0194, 0.0113, -0.1194]],
[[ 0.0527, -0.0886, 0.0359], [[ 0.0527, -0.0886, 0.0359],
[ 0.1595, 0.0526, -0.0048], [ 0.1595, 0.0526, -0.0048],
[-0.1790, -0.0458, -0.0324]]], [-0.1790, -0.0458, -0.0324]]],
[[[-0.1638, 0.0942, 0.0686], [[[-0.1638, 0.0942, 0.0686],
[-0.1082, -0.0675, 0.1892], [-0.1082, -0.0675, 0.1892],
[-0.1347, -0.1247, 0.0739]], [-0.1347, -0.1247, 0.0739]],
[[ 0.0595, 0.1504, -0.1657], [[ 0.0595, 0.1504, -0.1657],
[ 0.0733, 0.0529, -0.1599], [ 0.0733, 0.0529, -0.1599],
[ 0.0171, -0.1127, -0.0259]], [ 0.0171, -0.1127, -0.0259]],
[[-0.0092, 0.0193, 0.1176], [[-0.0092, 0.0193, 0.1176],
[-0.1183, 0.0101, 0.1011], [-0.1183, 0.0101, 0.1011],
[ 0.0648, -0.1897, 0.0782]]], [ 0.0648, -0.1897, 0.0782]]],
[[[ 0.0098, -0.1161, -0.0802], [[[ 0.0098, -0.1161, -0.0802],
[-0.1821, 0.0221, -0.1754], [-0.1821, 0.0221, -0.1754],
[-0.1218, 0.0525, -0.0480]], [-0.1218, 0.0525, -0.0480]],
[[ 0.0770, 0.0477, 0.1514], [[ 0.0770, 0.0477, 0.1514],
[ 0.0374, -0.1075, -0.1026], [ 0.0374, -0.1075, -0.1026],
[-0.0581, -0.1011, 0.1241]], [-0.0581, -0.1011, 0.1241]],
[[-0.0567, -0.0163, 0.0374], [[-0.0567, -0.0163, 0.0374],
[-0.1739, -0.0579, 0.0704], [-0.1739, -0.0579, 0.0704],
[ 0.1817, 0.1561, 0.1677]]], [ 0.1817, 0.1561, 0.1677]]],
[[[-0.0569, -0.0763, 0.0044], [[[-0.0569, -0.0763, 0.0044],
[-0.1133, 0.0813, 0.1477], [-0.1133, 0.0813, 0.1477],
[ 0.0836, 0.0483, -0.1800]], [ 0.0836, 0.0483, -0.1800]],
[[ 0.1343, -0.1590, 0.1177], [[ 0.1343, -0.1590, 0.1177],
[ 0.1071, -0.1647, -0.0646], [ 0.1071, -0.1647, -0.0646],
[ 0.1578, -0.1261, 0.0243]], [ 0.1578, -0.1261, 0.0243]],
[[-0.0424, -0.0241, -0.0988], [[-0.0424, -0.0241, -0.0988],
[ 0.0023, 0.0029, -0.0291], [ 0.0023, 0.0029, -0.0291],
[ 0.0415, -0.0557, 0.1427]]], [ 0.0415, -0.0557, 0.1427]]],
[[[-0.1028, 0.1054, 0.1658], [[[-0.1028, 0.1054, 0.1658],
[-0.0357, 0.1579, 0.1237], [-0.0357, 0.1579, 0.1237],
[ 0.0368, 0.0532, -0.1043]], [ 0.0368, 0.0532, -0.1043]],
[[-0.0369, 0.0575, -0.1023], [[-0.0369, 0.0575, -0.1023],
[ 0.0635, 0.1015, 0.1112], [ 0.0635, 0.1015, 0.1112],
[-0.1235, 0.0467, 0.0908]], [-0.1235, 0.0467, 0.0908]],
[[ 0.1380, 0.0633, 0.1087], [[ 0.1380, 0.0633, 0.1087],
[-0.1360, 0.0422, -0.1524], [-0.1360, 0.0422, -0.1524],
[ 0.0819, 0.0918, -0.1624]]], [ 0.0819, 0.0918, -0.1624]]],
[[[ 0.1584, -0.0218, -0.0236], [[[ 0.1584, -0.0218, -0.0236],
[ 0.1878, -0.1289, 0.1343], [ 0.1878, -0.1289, 0.1343],
[ 0.0351, 0.1225, -0.1460]], [ 0.0351, 0.1225, -0.1460]],
[[ 0.0690, -0.1439, 0.0056], [[ 0.0690, -0.1439, 0.0056],
[ 0.0272, -0.0058, 0.0125], [ 0.0272, -0.0058, 0.0125],
[ 0.0868, -0.0684, -0.0884]], [ 0.0868, -0.0684, -0.0884]],
[[ 0.1045, 0.0583, -0.0870], [[ 0.1045, 0.0583, -0.0870],
[ 0.0600, -0.0732, -0.1695], [ 0.0600, -0.0732, -0.1695],
[ 0.0953, 0.0246, 0.1245]]]])), [ 0.0953, 0.0246, 0.1245]]]])),
('conv1.bias', ('conv1.bias',
tensor([-1.8698e-01, -7.9379e-06, -1.9277e-02, 5.2182e-02, 7.5716e-02, tensor([-1.8698e-01, -7.9379e-06, -1.9277e-02, 5.2182e-02, 7.5716e-02,
-3.3830e-03, -9.6565e-02, 1.0241e-01, -8.2457e-02, -1.6224e-01, -3.3830e-03, -9.6565e-02, 1.0241e-01, -8.2457e-02, -1.6224e-01,
1.2980e-01, -8.2256e-02, -7.4655e-02, -3.7980e-02, 8.3407e-02, 1.2980e-01, -8.2256e-02, -7.4655e-02, -3.7980e-02, 8.3407e-02,
-1.4880e-01, 4.8939e-02, 2.7506e-02, 5.8676e-03, -1.5813e-01, -1.4880e-01, 4.8939e-02, 2.7506e-02, 5.8676e-03, -1.5813e-01,
-6.2464e-04, 1.0359e-02, -1.5525e-01, 7.9100e-02, 1.6850e-02, -6.2464e-04, 1.0359e-02, -1.5525e-01, 7.9100e-02, 1.6850e-02,
-1.3809e-01, -6.3393e-02, -5.3843e-02, -1.5219e-02, -1.7365e-01, -1.3809e-01, -6.3393e-02, -5.3843e-02, -1.5219e-02, -1.7365e-01,
1.7249e-01, -1.1165e-01])), 1.7249e-01, -1.1165e-01])),
('conv2.weight', ('conv2.weight',
tensor([[[[ 4.3750e-02, 4.5533e-02, -2.9410e-02], tensor([[[[ 4.3750e-02, 4.5533e-02, -2.9410e-02],
[-4.1395e-02, 5.0397e-04, -1.3265e-02], [-4.1395e-02, 5.0397e-04, -1.3265e-02],
[-4.9851e-02, -1.0518e-02, 5.7710e-02]], [-4.9851e-02, -1.0518e-02, 5.7710e-02]],
[[-5.6332e-02, -4.7168e-03, -4.4627e-02], [[-5.6332e-02, -4.7168e-03, -4.4627e-02],
[ 5.3513e-03, -4.0824e-02, 1.8281e-02], [ 5.3513e-03, -4.0824e-02, 1.8281e-02],
[ 5.0677e-02, -1.5295e-02, -6.1751e-03]], [ 5.0677e-02, -1.5295e-02, -6.1751e-03]],
[[-2.4984e-02, 1.2784e-02, -4.7123e-02], [[-2.4984e-02, 1.2784e-02, -4.7123e-02],
[-4.3238e-02, 4.7349e-02, -1.5219e-02], [-4.3238e-02, 4.7349e-02, -1.5219e-02],
[-3.6073e-02, 4.1506e-02, -3.5337e-02]], [-3.6073e-02, 4.1506e-02, -3.5337e-02]],
..., ...,
[[ 5.6048e-02, 1.9129e-03, -1.7200e-02], [[ 5.6048e-02, 1.9129e-03, -1.7200e-02],
[ 5.8869e-02, -5.1520e-02, -5.3205e-02], [ 5.8869e-02, -5.1520e-02, -5.3205e-02],
[-1.3903e-02, 5.1790e-02, 2.2585e-02]], [-1.3903e-02, 5.1790e-02, 2.2585e-02]],
[[ 1.1835e-02, -4.9313e-02, -3.1838e-02], [[ 1.1835e-02, -4.9313e-02, -3.1838e-02],
[ 7.6813e-03, 4.2715e-02, -5.7404e-02], [ 7.6813e-03, 4.2715e-02, -5.7404e-02],
[-4.1474e-02, -2.3128e-02, -4.7935e-02]], [-4.1474e-02, -2.3128e-02, -4.7935e-02]],
[[-2.1860e-02, -2.1817e-02, -3.2578e-02], [[-2.1860e-02, -2.1817e-02, -3.2578e-02],
[ 3.1317e-02, 3.3435e-02, 3.1837e-02], [ 3.1317e-02, 3.3435e-02, 3.1837e-02],
[-2.2399e-03, 3.1600e-02, 4.0183e-02]]], [-2.2399e-03, 3.1600e-02, 4.0183e-02]]],
[[[ 1.9610e-04, 5.3780e-02, -4.5810e-02], [[[ 1.9610e-04, 5.3780e-02, -4.5810e-02],
[ 4.0340e-02, 1.4904e-02, -1.5597e-02], [ 4.0340e-02, 1.4904e-02, -1.5597e-02],
[-4.6080e-02, 5.0714e-02, -5.7445e-03]], [-4.6080e-02, 5.0714e-02, -5.7445e-03]],
[[-3.5281e-02, 3.3011e-02, 4.3343e-02], [[-3.5281e-02, 3.3011e-02, 4.3343e-02],
[-4.6263e-02, -5.6184e-02, 5.1245e-03], [-4.6263e-02, -5.6184e-02, 5.1245e-03],
[ 3.6015e-02, -3.3152e-02, 4.6629e-03]], [ 3.6015e-02, -3.3152e-02, 4.6629e-03]],
[[ 1.7650e-03, -4.2336e-02, 4.3744e-02], [[ 1.7650e-03, -4.2336e-02, 4.3744e-02],
[ 2.1655e-02, 5.3759e-02, 1.3719e-03], [ 2.1655e-02, 5.3759e-02, 1.3719e-03],
[ 4.2005e-02, 5.3998e-02, 1.9009e-02]], [ 4.2005e-02, 5.3998e-02, 1.9009e-02]],
..., ...,
[[ 8.9786e-03, -1.8645e-02, -1.3587e-02], [[ 8.9786e-03, -1.8645e-02, -1.3587e-02],
[-5.4167e-02, 1.2335e-02, -3.0384e-02], [-5.4167e-02, 1.2335e-02, -3.0384e-02],
[-4.8722e-03, -3.7296e-02, -2.6446e-02]], [-4.8722e-03, -3.7296e-02, -2.6446e-02]],
[[ 1.7580e-02, 3.8462e-02, -5.0269e-02], [[ 1.7580e-02, 3.8462e-02, -5.0269e-02],
[ 2.6601e-03, -1.1462e-02, 4.7459e-02], [ 2.6601e-03, -1.1462e-02, 4.7459e-02],
[-2.8888e-02, 3.4436e-02, -4.9943e-02]], [-2.8888e-02, 3.4436e-02, -4.9943e-02]],
[[-5.0206e-02, -5.6025e-02, -3.6346e-02], [[-5.0206e-02, -5.6025e-02, -3.6346e-02],
[-2.4407e-02, 5.3721e-02, -5.4920e-02], [-2.4407e-02, 5.3721e-02, -5.4920e-02],
[ 5.1835e-02, -3.2396e-02, 3.2373e-02]]], [ 5.1835e-02, -3.2396e-02, 3.2373e-02]]],
[[[-2.7759e-02, -5.4737e-02, -1.1689e-02], [[[-2.7759e-02, -5.4737e-02, -1.1689e-02],
[ 3.9462e-02, 2.8649e-02, 5.1776e-02], [ 3.9462e-02, 2.8649e-02, 5.1776e-02],
[ 2.4253e-02, -2.8318e-02, 2.7402e-02]], [ 2.4253e-02, -2.8318e-02, 2.7402e-02]],
[[ 1.3045e-02, -1.0456e-02, 2.0426e-02], [[ 1.3045e-02, -1.0456e-02, 2.0426e-02],
[ 2.1949e-02, 4.6817e-02, -5.6093e-02], [ 2.1949e-02, 4.6817e-02, -5.6093e-02],
[ 2.7145e-02, -5.5441e-02, -2.0719e-02]], [ 2.7145e-02, -5.5441e-02, -2.0719e-02]],
[[ 4.4704e-02, -2.4099e-02, -4.7185e-02], [[ 4.4704e-02, -2.4099e-02, -4.7185e-02],
[-4.3257e-02, -3.3058e-02, -8.6451e-03], [-4.3257e-02, -3.3058e-02, -8.6451e-03],
[-3.7283e-02, -3.4569e-02, -7.1049e-03]], [-3.7283e-02, -3.4569e-02, -7.1049e-03]],
..., ...,
[[-4.1559e-02, -2.9240e-02, 2.7197e-03], [[-4.1559e-02, -2.9240e-02, 2.7197e-03],
[ 2.0770e-02, 5.4479e-02, -4.4845e-02], [ 2.0770e-02, 5.4479e-02, -4.4845e-02],
[-1.1641e-02, -2.9814e-02, -2.4419e-02]], [-1.1641e-02, -2.9814e-02, -2.4419e-02]],
[[-1.5743e-02, 1.0854e-02, 3.0878e-02], [[-1.5743e-02, 1.0854e-02, 3.0878e-02],
[ 2.2739e-02, 3.2999e-02, -1.1902e-02], [ 2.2739e-02, 3.2999e-02, -1.1902e-02],
[-3.4837e-02, 1.5305e-02, -8.7552e-03]], [-3.4837e-02, 1.5305e-02, -8.7552e-03]],
[[-2.2882e-02, 9.4639e-03, 5.1878e-03], [[-2.2882e-02, 9.4639e-03, 5.1878e-03],
[-2.6344e-02, 2.9063e-02, -1.9337e-02], [-2.6344e-02, 2.9063e-02, -1.9337e-02],
[-3.4314e-02, 1.5313e-02, 4.1524e-02]]], [-3.4314e-02, 1.5313e-02, 4.1524e-02]]],
..., ...,
[[[ 9.7454e-03, -3.2903e-03, 1.0696e-02], [[[ 9.7454e-03, -3.2903e-03, 1.0696e-02],
[-4.0918e-02, 1.6352e-02, 1.4646e-02], [-4.0918e-02, 1.6352e-02, 1.4646e-02],
[ 1.2516e-02, -2.1804e-02, -2.5489e-02]], [ 1.2516e-02, -2.1804e-02, -2.5489e-02]],
[[-1.6083e-02, 2.5374e-02, 3.1458e-02], [[-1.6083e-02, 2.5374e-02, 3.1458e-02],
[-3.1497e-02, -1.9513e-02, -2.1223e-02], [-3.1497e-02, -1.9513e-02, -2.1223e-02],
[ 6.6286e-03, 1.6538e-02, -4.8944e-02]], [ 6.6286e-03, 1.6538e-02, -4.8944e-02]],
[[ 2.4808e-02, -2.9520e-02, -4.8227e-02], [[ 2.4808e-02, -2.9520e-02, -4.8227e-02],
[ 1.7325e-03, -4.7443e-02, 2.3087e-03], [ 1.7325e-03, -4.7443e-02, 2.3087e-03],
[-1.0008e-02, -2.0313e-02, 2.9944e-02]], [-1.0008e-02, -2.0313e-02, 2.9944e-02]],
..., ...,
[[ 5.5781e-02, -2.0898e-02, -3.1487e-03], [[ 5.5781e-02, -2.0898e-02, -3.1487e-03],
[-1.6931e-02, 4.5279e-04, -1.5024e-02], [-1.6931e-02, 4.5279e-04, -1.5024e-02],
[-5.5885e-02, 2.7140e-02, -8.5434e-03]], [-5.5885e-02, 2.7140e-02, -8.5434e-03]],
[[ 1.3970e-02, -3.3131e-02, 4.3112e-02], [[ 1.3970e-02, -3.3131e-02, 4.3112e-02],
[-3.4956e-02, -5.0144e-02, -1.6391e-02], [-3.4956e-02, -5.0144e-02, -1.6391e-02],
[-9.1003e-03, -2.0204e-02, -1.0226e-03]], [-9.1003e-03, -2.0204e-02, -1.0226e-03]],
[[-4.0053e-02, -5.0194e-02, 5.0405e-02], [[-4.0053e-02, -5.0194e-02, 5.0405e-02],
[ 5.4107e-02, 4.2185e-02, 3.4359e-02], [ 5.4107e-02, 4.2185e-02, 3.4359e-02],
[ 1.6749e-02, -1.4102e-02, 5.0171e-02]]], [ 1.6749e-02, -1.4102e-02, 5.0171e-02]]],
[[[ 2.8229e-02, 5.6586e-02, -3.9617e-03], [[[ 2.8229e-02, 5.6586e-02, -3.9617e-03],
[ 2.9538e-02, -1.2507e-02, -2.5516e-02], [ 2.9538e-02, -1.2507e-02, -2.5516e-02],
[-1.5193e-02, -2.9232e-02, -2.0701e-02]], [-1.5193e-02, -2.9232e-02, -2.0701e-02]],
[[-5.8773e-02, 3.3015e-02, -9.4146e-03], [[-5.8773e-02, 3.3015e-02, -9.4146e-03],
[ 2.8957e-02, 5.8666e-02, 2.8679e-02], [ 2.8957e-02, 5.8666e-02, 2.8679e-02],
[ 1.5249e-02, -1.2246e-03, 1.2230e-03]], [ 1.5249e-02, -1.2246e-03, 1.2230e-03]],
[[ 2.6050e-02, -4.6042e-02, -3.4895e-03], [[ 2.6050e-02, -4.6042e-02, -3.4895e-03],
[ 4.9529e-02, 6.6835e-03, -4.1808e-02], [ 4.9529e-02, 6.6835e-03, -4.1808e-02],
[-8.6450e-03, -4.8510e-02, -2.4011e-02]], [-8.6450e-03, -4.8510e-02, -2.4011e-02]],
..., ...,
[[-5.1427e-02, 2.4528e-02, -5.4878e-02], [[-5.1427e-02, 2.4528e-02, -5.4878e-02],
[-1.8610e-02, 5.4365e-02, 3.5053e-03], [-1.8610e-02, 5.4365e-02, 3.5053e-03],
[-3.9922e-02, 4.2510e-02, -5.7261e-02]], [-3.9922e-02, 4.2510e-02, -5.7261e-02]],
[[ 4.1938e-02, -4.2039e-02, -1.2487e-02], [[ 4.1938e-02, -4.2039e-02, -1.2487e-02],
[-1.4090e-02, -3.7895e-02, 1.4394e-02], [-1.4090e-02, -3.7895e-02, 1.4394e-02],
[ 2.2555e-02, -2.7264e-02, 5.6102e-02]], [ 2.2555e-02, -2.7264e-02, 5.6102e-02]],
[[ 1.5770e-02, 5.4672e-02, -2.4056e-02], [[ 1.5770e-02, 5.4672e-02, -2.4056e-02],
[ 5.2089e-02, -2.8859e-02, -2.6499e-03], [ 5.2089e-02, -2.8859e-02, -2.6499e-03],
[-5.2122e-02, -3.7436e-02, 3.9897e-02]]], [-5.2122e-02, -3.7436e-02, 3.9897e-02]]],
[[[ 2.7888e-02, 2.9241e-02, -1.9488e-02], [[[ 2.7888e-02, 2.9241e-02, -1.9488e-02],
[ 2.8928e-02, 5.3312e-02, -2.9810e-02], [ 2.8928e-02, 5.3312e-02, -2.9810e-02],
[-8.5104e-03, 5.7751e-02, -8.1857e-03]], [-8.5104e-03, 5.7751e-02, -8.1857e-03]],
[[ 4.2649e-02, 3.3158e-03, 4.2879e-02], [[ 4.2649e-02, 3.3158e-03, 4.2879e-02],
[ 7.7893e-03, -3.2879e-02, 2.7630e-02], [ 7.7893e-03, -3.2879e-02, 2.7630e-02],
[ 5.4706e-03, 4.8019e-02, 1.2420e-02]], [ 5.4706e-03, 4.8019e-02, 1.2420e-02]],
[[-4.2004e-02, -4.2790e-02, 2.4634e-02], [[-4.2004e-02, -4.2790e-02, 2.4634e-02],
[-5.4641e-02, 3.4600e-02, 2.9071e-03], [-5.4641e-02, 3.4600e-02, 2.9071e-03],
[ 2.6470e-02, 4.6701e-02, 3.7158e-02]], [ 2.6470e-02, 4.6701e-02, 3.7158e-02]],
..., ...,
[[-1.7641e-02, -2.1205e-02, -5.1504e-02], [[-1.7641e-02, -2.1205e-02, -5.1504e-02],
[-7.4737e-03, 5.5061e-02, -2.6397e-03], [-7.4737e-03, 5.5061e-02, -2.6397e-03],
[-4.4653e-02, -3.6719e-02, 3.4420e-06]], [-4.4653e-02, -3.6719e-02, 3.4420e-06]],
[[ 1.6525e-02, 1.7280e-02, 5.4554e-03], [[ 1.6525e-02, 1.7280e-02, 5.4554e-03],
[ 4.0098e-02, 2.7571e-02, -4.4965e-02], [ 4.0098e-02, 2.7571e-02, -4.4965e-02],
[ 6.1493e-03, -5.7754e-02, 1.0513e-02]], [ 6.1493e-03, -5.7754e-02, 1.0513e-02]],
[[-5.7615e-02, 3.2921e-02, -1.5900e-02], [[-5.7615e-02, 3.2921e-02, -1.5900e-02],
[ 2.0081e-02, 5.4590e-02, 1.1296e-02], [ 2.0081e-02, 5.4590e-02, 1.1296e-02],
[-4.5015e-02, 1.1341e-03, 2.6447e-02]]]])), [-4.5015e-02, 1.1341e-03, 2.6447e-02]]]])),
('conv2.bias', ('conv2.bias',
tensor([-0.0538, -0.0320, -0.0153, 0.0558, 0.0254, 0.0281, -0.0148, 0.0060, tensor([-0.0538, -0.0320, -0.0153, 0.0558, 0.0254, 0.0281, -0.0148, 0.0060,
-0.0283, -0.0062, 0.0437, -0.0064, 0.0341, 0.0233, -0.0201, 0.0391, -0.0283, -0.0062, 0.0437, -0.0064, 0.0341, 0.0233, -0.0201, 0.0391,
0.0243, 0.0071, 0.0125, -0.0138, -0.0377, -0.0169, -0.0475, -0.0004, 0.0243, 0.0071, 0.0125, -0.0138, -0.0377, -0.0169, -0.0475, -0.0004,
-0.0105, -0.0502, 0.0241, 0.0090, 0.0069, -0.0315, -0.0192, 0.0204])), -0.0105, -0.0502, 0.0241, 0.0090, 0.0069, -0.0315, -0.0192, 0.0204])),
('conv3.weight', ('conv3.weight',
tensor([[[[-0.0215, -0.0208, -0.0272], tensor([[[[-0.0215, -0.0208, -0.0272],
[-0.0493, -0.0117, -0.0285], [-0.0493, -0.0117, -0.0285],
[ 0.0515, -0.0041, 0.0126]], [ 0.0515, -0.0041, 0.0126]],
[[ 0.0299, -0.0301, 0.0552], [[ 0.0299, -0.0301, 0.0552],
[ 0.0450, 0.0449, -0.0583], [ 0.0450, 0.0449, -0.0583],
[-0.0452, -0.0480, -0.0275]], [-0.0452, -0.0480, -0.0275]],
[[-0.0262, -0.0338, 0.0505], [[-0.0262, -0.0338, 0.0505],
[ 0.0146, -0.0364, -0.0044], [ 0.0146, -0.0364, -0.0044],
[-0.0102, -0.0051, 0.0017]], [-0.0102, -0.0051, 0.0017]],
..., ...,
[[-0.0367, -0.0468, -0.0586], [[-0.0367, -0.0468, -0.0586],
[ 0.0126, 0.0037, 0.0191], [ 0.0126, 0.0037, 0.0191],
[-0.0153, 0.0048, -0.0160]], [-0.0153, 0.0048, -0.0160]],
[[-0.0050, 0.0364, 0.0582], [[-0.0050, 0.0364, 0.0582],
[ 0.0093, -0.0268, -0.0355], [ 0.0093, -0.0268, -0.0355],
[-0.0125, 0.0500, 0.0009]], [-0.0125, 0.0500, 0.0009]],
[[ 0.0237, -0.0211, -0.0130], [[ 0.0237, -0.0211, -0.0130],
[-0.0489, 0.0118, 0.0387], [-0.0489, 0.0118, 0.0387],
[-0.0006, 0.0301, 0.0283]]], [-0.0006, 0.0301, 0.0283]]],
[[[-0.0391, -0.0464, -0.0158], [[[-0.0391, -0.0464, -0.0158],
[ 0.0201, -0.0054, 0.0422], [ 0.0201, -0.0054, 0.0422],
[ 0.0085, -0.0474, -0.0251]], [ 0.0085, -0.0474, -0.0251]],
[[-0.0346, 0.0536, -0.0391], [[-0.0346, 0.0536, -0.0391],
[ 0.0244, -0.0263, -0.0073], [ 0.0244, -0.0263, -0.0073],
[ 0.0076, 0.0160, 0.0044]], [ 0.0076, 0.0160, 0.0044]],
[[-0.0128, 0.0146, -0.0381], [[-0.0128, 0.0146, -0.0381],
[-0.0277, -0.0142, 0.0226], [-0.0277, -0.0142, 0.0226],
[ 0.0190, 0.0326, -0.0219]], [ 0.0190, 0.0326, -0.0219]],
..., ...,
[[-0.0217, 0.0129, 0.0558], [[-0.0217, 0.0129, 0.0558],
[ 0.0164, -0.0292, -0.0467], [ 0.0164, -0.0292, -0.0467],
[-0.0296, 0.0205, -0.0300]], [-0.0296, 0.0205, -0.0300]],
[[ 0.0254, -0.0151, -0.0583], [[ 0.0254, -0.0151, -0.0583],
[ 0.0111, -0.0469, -0.0300], [ 0.0111, -0.0469, -0.0300],
[-0.0462, 0.0293, -0.0351]], [-0.0462, 0.0293, -0.0351]],
[[ 0.0401, -0.0251, 0.0160], [[ 0.0401, -0.0251, 0.0160],
[-0.0160, -0.0195, -0.0065], [-0.0160, -0.0195, -0.0065],
[-0.0519, 0.0351, 0.0357]]], [-0.0519, 0.0351, 0.0357]]],
[[[ 0.0544, -0.0209, -0.0454], [[[ 0.0544, -0.0209, -0.0454],
[ 0.0287, -0.0205, -0.0294], [ 0.0287, -0.0205, -0.0294],
[-0.0195, -0.0235, -0.0378]], [-0.0195, -0.0235, -0.0378]],
[[-0.0294, -0.0380, 0.0301], [[-0.0294, -0.0380, 0.0301],
[ 0.0360, 0.0367, 0.0458], [ 0.0360, 0.0367, 0.0458],
[-0.0189, -0.0017, 0.0145]], [-0.0189, -0.0017, 0.0145]],
[[-0.0297, 0.0567, 0.0276], [[-0.0297, 0.0567, 0.0276],
[ 0.0298, 0.0383, 0.0227], [ 0.0298, 0.0383, 0.0227],
[ 0.0262, 0.0063, 0.0131]], [ 0.0262, 0.0063, 0.0131]],
..., ...,
[[ 0.0002, 0.0432, -0.0247], [[ 0.0002, 0.0432, -0.0247],
[ 0.0068, -0.0298, -0.0484], [ 0.0068, -0.0298, -0.0484],
[-0.0361, -0.0014, 0.0444]], [-0.0361, -0.0014, 0.0444]],
[[-0.0184, -0.0201, -0.0163], [[-0.0184, -0.0201, -0.0163],
[-0.0466, 0.0255, -0.0244], [-0.0466, 0.0255, -0.0244],
[ 0.0283, 0.0149, -0.0588]], [ 0.0283, 0.0149, -0.0588]],
[[ 0.0323, 0.0392, -0.0254], [[ 0.0323, 0.0392, -0.0254],
[ 0.0560, 0.0137, -0.0401], [ 0.0560, 0.0137, -0.0401],
[-0.0236, 0.0589, 0.0448]]], [-0.0236, 0.0589, 0.0448]]],
..., ...,
[[[ 0.0213, 0.0204, 0.0574], [[[ 0.0213, 0.0204, 0.0574],
[-0.0276, -0.0196, 0.0117], [-0.0276, -0.0196, 0.0117],
[ 0.0569, -0.0158, -0.0502]], [ 0.0569, -0.0158, -0.0502]],
[[ 0.0452, -0.0038, 0.0502], [[ 0.0452, -0.0038, 0.0502],
[ 0.0428, -0.0398, -0.0486], [ 0.0428, -0.0398, -0.0486],
[ 0.0130, 0.0563, 0.0576]], [ 0.0130, 0.0563, 0.0576]],
[[ 0.0484, -0.0535, 0.0048], [[ 0.0484, -0.0535, 0.0048],
[ 0.0268, -0.0290, -0.0390], [ 0.0268, -0.0290, -0.0390],
[ 0.0189, -0.0194, -0.0588]], [ 0.0189, -0.0194, -0.0588]],
..., ...,
[[ 0.0163, -0.0113, -0.0520], [[ 0.0163, -0.0113, -0.0520],
[ 0.0288, -0.0547, -0.0544], [ 0.0288, -0.0547, -0.0544],
[ 0.0442, 0.0376, 0.0566]], [ 0.0442, 0.0376, 0.0566]],
[[-0.0343, 0.0569, 0.0438], [[-0.0343, 0.0569, 0.0438],
[-0.0403, -0.0372, -0.0532], [-0.0403, -0.0372, -0.0532],
[ 0.0322, 0.0126, 0.0423]], [ 0.0322, 0.0126, 0.0423]],
[[ 0.0577, 0.0136, -0.0480], [[ 0.0577, 0.0136, -0.0480],
[-0.0293, -0.0348, 0.0342], [-0.0293, -0.0348, 0.0342],
[-0.0510, -0.0078, -0.0042]]], [-0.0510, -0.0078, -0.0042]]],
[[[-0.0243, 0.0406, 0.0537], [[[-0.0243, 0.0406, 0.0537],
[ 0.0209, -0.0059, -0.0487], [ 0.0209, -0.0059, -0.0487],
[-0.0425, 0.0339, 0.0444]], [-0.0425, 0.0339, 0.0444]],
[[ 0.0465, -0.0467, 0.0461], [[ 0.0465, -0.0467, 0.0461],
[-0.0389, 0.0144, -0.0502], [-0.0389, 0.0144, -0.0502],
[ 0.0274, 0.0552, 0.0356]], [ 0.0274, 0.0552, 0.0356]],
[[-0.0289, 0.0474, -0.0217], [[-0.0289, 0.0474, -0.0217],
[ 0.0472, -0.0135, 0.0164], [ 0.0472, -0.0135, 0.0164],
[-0.0165, -0.0049, -0.0475]], [-0.0165, -0.0049, -0.0475]],
..., ...,
[[-0.0377, 0.0267, 0.0367], [[-0.0377, 0.0267, 0.0367],
[ 0.0111, 0.0114, -0.0329], [ 0.0111, 0.0114, -0.0329],
[ 0.0031, -0.0223, -0.0280]], [ 0.0031, -0.0223, -0.0280]],
[[-0.0500, -0.0529, 0.0116], [[-0.0500, -0.0529, 0.0116],
[ 0.0483, 0.0121, -0.0149], [ 0.0483, 0.0121, -0.0149],
[ 0.0328, 0.0201, 0.0402]], [ 0.0328, 0.0201, 0.0402]],
[[ 0.0463, 0.0157, -0.0332], [[ 0.0463, 0.0157, -0.0332],
[ 0.0150, 0.0479, 0.0461], [ 0.0150, 0.0479, 0.0461],
[ 0.0275, 0.0506, -0.0466]]], [ 0.0275, 0.0506, -0.0466]]],
[[[-0.0478, 0.0274, 0.0500], [[[-0.0478, 0.0274, 0.0500],
[-0.0394, 0.0032, -0.0496], [-0.0394, 0.0032, -0.0496],
[ 0.0381, 0.0391, 0.0330]], [ 0.0381, 0.0391, 0.0330]],
[[ 0.0184, -0.0560, -0.0345], [[ 0.0184, -0.0560, -0.0345],
[-0.0459, -0.0215, 0.0452], [-0.0459, -0.0215, 0.0452],
[ 0.0049, 0.0537, 0.0544]], [ 0.0049, 0.0537, 0.0544]],
[[-0.0413, -0.0084, 0.0585], [[-0.0413, -0.0084, 0.0585],
[ 0.0338, -0.0067, -0.0113], [ 0.0338, -0.0067, -0.0113],
[-0.0187, -0.0234, -0.0525]], [-0.0187, -0.0234, -0.0525]],
..., ...,
[[-0.0389, 0.0325, -0.0538], [[-0.0389, 0.0325, -0.0538],
[ 0.0118, 0.0509, 0.0352], [ 0.0118, 0.0509, 0.0352],
[-0.0351, -0.0341, -0.0506]], [-0.0351, -0.0341, -0.0506]],
[[ 0.0136, -0.0349, 0.0082], [[ 0.0136, -0.0349, 0.0082],
[ 0.0358, -0.0211, 0.0537], [ 0.0358, -0.0211, 0.0537],
[-0.0183, 0.0390, -0.0267]], [-0.0183, 0.0390, -0.0267]],
[[-0.0219, -0.0145, -0.0351], [[-0.0219, -0.0145, -0.0351],
[ 0.0556, 0.0033, -0.0030], [ 0.0556, 0.0033, -0.0030],
[ 0.0075, -0.0425, -0.0365]]]])), [ 0.0075, -0.0425, -0.0365]]]])),
('conv3.bias', ('conv3.bias',
tensor([ 0.0304, 0.0099, -0.0004, 0.0334, 0.0301, 0.0491, 0.0530, -0.0432, tensor([ 0.0304, 0.0099, -0.0004, 0.0334, 0.0301, 0.0491, 0.0530, -0.0432,
-0.0127, -0.0549, -0.0419, 0.0159, -0.0284, 0.0295, -0.0148, 0.0275, -0.0127, -0.0549, -0.0419, 0.0159, -0.0284, 0.0295, -0.0148, 0.0275,
0.0554, -0.0056, 0.0389, -0.0264, -0.0383, 0.0126, 0.0320, 0.0312, 0.0554, -0.0056, 0.0389, -0.0264, -0.0383, 0.0126, 0.0320, 0.0312,
0.0018, 0.0560, -0.0329, -0.0155, -0.0391, -0.0539, -0.0571, -0.0254])), 0.0018, 0.0560, -0.0329, -0.0155, -0.0391, -0.0539, -0.0571, -0.0254])),
('conv4.weight', ('conv4.weight',
tensor([[[[-3.8911e-02, -3.4220e-02, 4.2567e-03], tensor([[[[-3.8911e-02, -3.4220e-02, 4.2567e-03],
[-4.5321e-02, -5.2531e-02, -8.1722e-03], [-4.5321e-02, -5.2531e-02, -8.1722e-03],
[-2.2638e-02, 4.4213e-02, 5.6989e-02]], [-2.2638e-02, 4.4213e-02, 5.6989e-02]],
[[ 1.8417e-03, -1.4453e-02, 4.9892e-02], [[ 1.8417e-03, -1.4453e-02, 4.9892e-02],
[ 5.7762e-02, 9.6610e-03, -3.9509e-02], [ 5.7762e-02, 9.6610e-03, -3.9509e-02],
[ 3.3795e-02, 5.0409e-02, -5.8834e-02]], [ 3.3795e-02, 5.0409e-02, -5.8834e-02]],
[[ 4.6645e-03, -1.6286e-02, 4.3410e-02], [[ 4.6645e-03, -1.6286e-02, 4.3410e-02],
[-3.4043e-02, -2.2207e-02, 4.0967e-02], [-3.4043e-02, -2.2207e-02, 4.0967e-02],
[ 5.3004e-02, -2.2756e-02, -6.7993e-03]], [ 5.3004e-02, -2.2756e-02, -6.7993e-03]],
..., ...,
[[ 1.1741e-02, -5.5062e-02, -3.3625e-02], [[ 1.1741e-02, -5.5062e-02, -3.3625e-02],
[-9.2320e-03, -3.3036e-02, 3.3196e-02], [-9.2320e-03, -3.3036e-02, 3.3196e-02],
[ 2.3940e-02, 2.0442e-02, 1.4183e-02]], [ 2.3940e-02, 2.0442e-02, 1.4183e-02]],
[[-2.7139e-02, -3.4129e-03, -1.0090e-02], [[-2.7139e-02, -3.4129e-03, -1.0090e-02],
[ 1.3073e-02, -1.6998e-02, -4.7540e-02], [ 1.3073e-02, -1.6998e-02, -4.7540e-02],
[-2.5758e-02, -1.9363e-02, 2.1905e-02]], [-2.5758e-02, -1.9363e-02, 2.1905e-02]],
[[ 5.7593e-02, 1.5013e-02, -5.7894e-02], [[ 5.7593e-02, 1.5013e-02, -5.7894e-02],
[ 5.7964e-02, -2.3412e-02, 2.6955e-02], [ 5.7964e-02, -2.3412e-02, 2.6955e-02],
[-3.9814e-02, -4.6015e-02, -5.3240e-02]]], [-3.9814e-02, -4.6015e-02, -5.3240e-02]]],
[[[ 5.8211e-02, -4.1118e-02, 2.7704e-02], [[[ 5.8211e-02, -4.1118e-02, 2.7704e-02],
[ 5.7198e-02, 8.4165e-03, -5.1708e-02], [ 5.7198e-02, 8.4165e-03, -5.1708e-02],
[ 3.1423e-02, 1.5026e-02, 3.5922e-02]], [ 3.1423e-02, 1.5026e-02, 3.5922e-02]],
[[ 8.8858e-03, 3.2818e-02, 5.4486e-02], [[ 8.8858e-03, 3.2818e-02, 5.4486e-02],
[-2.6636e-02, 2.2604e-02, 2.9531e-02], [-2.6636e-02, 2.2604e-02, 2.9531e-02],
[-1.0327e-03, 2.2348e-03, 2.4103e-02]], [-1.0327e-03, 2.2348e-03, 2.4103e-02]],
[[ 3.8683e-02, -5.0057e-03, 5.0224e-02], [[ 3.8683e-02, -5.0057e-03, 5.0224e-02],
[ 3.5756e-02, -2.7295e-02, -2.2854e-02], [ 3.5756e-02, -2.7295e-02, -2.2854e-02],
[-3.2043e-02, -3.2415e-02, 4.1034e-02]], [-3.2043e-02, -3.2415e-02, 4.1034e-02]],
..., ...,
[[-1.9791e-02, 4.3243e-02, -3.5177e-02], [[-1.9791e-02, 4.3243e-02, -3.5177e-02],
[ 2.4554e-02, 4.2845e-03, 4.8009e-02], [ 2.4554e-02, 4.2845e-03, 4.8009e-02],
[ 2.4897e-03, 3.9550e-02, -3.0833e-02]], [ 2.4897e-03, 3.9550e-02, -3.0833e-02]],
[[ 4.5807e-02, 6.5845e-03, 9.3362e-05], [[ 4.5807e-02, 6.5845e-03, 9.3362e-05],
[-1.9411e-02, -2.9161e-02, 5.0828e-02], [-1.9411e-02, -2.9161e-02, 5.0828e-02],
[ 1.2028e-03, 2.1260e-02, -4.3710e-03]], [ 1.2028e-03, 2.1260e-02, -4.3710e-03]],
[[-4.8702e-02, -2.0571e-02, -3.5162e-02], [[-4.8702e-02, -2.0571e-02, -3.5162e-02],
[-2.5856e-02, -2.6619e-02, 8.1867e-03], [-2.5856e-02, -2.6619e-02, 8.1867e-03],
[-2.7671e-02, -9.6651e-03, -5.3279e-02]]], [-2.7671e-02, -9.6651e-03, -5.3279e-02]]],
[[[-5.6432e-02, 2.3722e-02, -2.1750e-02], [[[-5.6432e-02, 2.3722e-02, -2.1750e-02],
[-4.8247e-03, -2.1226e-02, -1.0829e-02], [-4.8247e-03, -2.1226e-02, -1.0829e-02],
[-1.9523e-02, -1.8187e-02, 2.2772e-03]], [-1.9523e-02, -1.8187e-02, 2.2772e-03]],
[[-1.0907e-02, 3.3984e-02, -1.2088e-02], [[-1.0907e-02, 3.3984e-02, -1.2088e-02],
[-1.8657e-02, -4.8297e-02, 2.3614e-02], [-1.8657e-02, -4.8297e-02, 2.3614e-02],
[-3.9670e-02, 6.1733e-03, -2.9168e-02]], [-3.9670e-02, 6.1733e-03, -2.9168e-02]],
[[-4.2112e-02, 2.8203e-02, -1.7385e-03], [[-4.2112e-02, 2.8203e-02, -1.7385e-03],
[-2.5282e-02, -9.4592e-05, 6.5093e-03], [-2.5282e-02, -9.4592e-05, 6.5093e-03],
[-4.1745e-02, 4.3988e-03, -1.1622e-02]], [-4.1745e-02, 4.3988e-03, -1.1622e-02]],
..., ...,
[[ 4.2991e-02, 1.6912e-02, -4.3689e-02], [[ 4.2991e-02, 1.6912e-02, -4.3689e-02],
[ 5.1871e-02, 4.8566e-02, 3.6205e-02], [ 5.1871e-02, 4.8566e-02, 3.6205e-02],
[-3.2016e-02, -1.3596e-02, -2.7950e-02]], [-3.2016e-02, -1.3596e-02, -2.7950e-02]],
[[-2.8307e-02, -4.0278e-02, 1.5087e-02], [[-2.8307e-02, -4.0278e-02, 1.5087e-02],
[ 4.0443e-02, -3.5727e-02, 3.7196e-02], [ 4.0443e-02, -3.5727e-02, 3.7196e-02],
[-1.4194e-02, -2.7319e-02, -5.1305e-02]], [-1.4194e-02, -2.7319e-02, -5.1305e-02]],
[[-2.9962e-02, 2.4693e-02, -4.4912e-02], [[-2.9962e-02, 2.4693e-02, -4.4912e-02],
[ 5.5890e-03, 4.6671e-02, 3.3599e-02], [ 5.5890e-03, 4.6671e-02, 3.3599e-02],
[-3.9949e-02, -4.4716e-02, 2.2345e-02]]], [-3.9949e-02, -4.4716e-02, 2.2345e-02]]],
..., ...,
[[[ 3.1920e-02, -4.9932e-02, -1.0871e-02], [[[ 3.1920e-02, -4.9932e-02, -1.0871e-02],
[-3.7500e-02, 4.1638e-02, -1.3246e-02], [-3.7500e-02, 4.1638e-02, -1.3246e-02],
[ 1.6447e-02, -5.6741e-02, -3.7524e-02]], [ 1.6447e-02, -5.6741e-02, -3.7524e-02]],
[[ 3.3903e-02, -3.1321e-02, -4.4877e-02], [[ 3.3903e-02, -3.1321e-02, -4.4877e-02],
[-2.2473e-02, -2.4225e-02, 4.5838e-02], [-2.2473e-02, -2.4225e-02, 4.5838e-02],
[-2.0069e-02, 3.8338e-02, 5.8010e-02]], [-2.0069e-02, 3.8338e-02, 5.8010e-02]],
[[ 1.7602e-02, -5.2530e-02, 4.9331e-02], [[ 1.7602e-02, -5.2530e-02, 4.9331e-02],
[ 2.4509e-02, 2.3943e-02, -2.1774e-02], [ 2.4509e-02, 2.3943e-02, -2.1774e-02],
[-5.7154e-02, 5.7090e-02, 3.7531e-02]], [-5.7154e-02, 5.7090e-02, 3.7531e-02]],
..., ...,
[[ 2.8630e-02, -4.8644e-04, -5.3822e-02], [[ 2.8630e-02, -4.8644e-04, -5.3822e-02],
[-1.1102e-02, 4.8524e-02, -2.7142e-02], [-1.1102e-02, 4.8524e-02, -2.7142e-02],
[-5.3463e-02, 3.5607e-02, -1.1110e-02]], [-5.3463e-02, 3.5607e-02, -1.1110e-02]],
[[ 4.7891e-02, -3.4098e-02, 3.6984e-02], [[ 4.7891e-02, -3.4098e-02, 3.6984e-02],
[ 3.9062e-02, -5.1119e-03, -3.3252e-02], [ 3.9062e-02, -5.1119e-03, -3.3252e-02],
[ 5.5029e-02, 3.1092e-03, -5.9391e-03]], [ 5.5029e-02, 3.1092e-03, -5.9391e-03]],
[[ 9.6775e-03, 2.2903e-02, -1.5971e-02], [[ 9.6775e-03, 2.2903e-02, -1.5971e-02],
[-2.6969e-02, 8.5069e-04, -2.5744e-02], [-2.6969e-02, 8.5069e-04, -2.5744e-02],
[ 2.7311e-03, -2.2119e-02, 2.4367e-03]]], [ 2.7311e-03, -2.2119e-02, 2.4367e-03]]],
[[[ 1.5206e-02, -5.3504e-02, -1.8013e-02], [[[ 1.5206e-02, -5.3504e-02, -1.8013e-02],
[-3.0064e-02, 3.2887e-02, 1.7612e-02], [-3.0064e-02, 3.2887e-02, 1.7612e-02],
[-4.2775e-02, -2.7335e-02, 5.1532e-02]], [-4.2775e-02, -2.7335e-02, 5.1532e-02]],
[[ 4.3325e-02, 1.2909e-03, 5.6831e-02], [[ 4.3325e-02, 1.2909e-03, 5.6831e-02],
[ 4.8283e-03, -4.4274e-02, 1.7624e-02], [ 4.8283e-03, -4.4274e-02, 1.7624e-02],
[-7.2574e-03, 1.3743e-02, -5.1502e-02]], [-7.2574e-03, 1.3743e-02, -5.1502e-02]],
[[-1.4299e-02, -3.0024e-02, 4.9578e-02], [[-1.4299e-02, -3.0024e-02, 4.9578e-02],
[-4.6143e-02, -3.0686e-02, -5.0727e-02], [-4.6143e-02, -3.0686e-02, -5.0727e-02],
[ 5.4766e-02, -1.4012e-02, 3.3267e-02]], [ 5.4766e-02, -1.4012e-02, 3.3267e-02]],
..., ...,
[[ 5.2773e-02, 4.0294e-02, -2.6113e-02], [[ 5.2773e-02, 4.0294e-02, -2.6113e-02],
[-2.5069e-02, 4.3956e-02, -4.7841e-02], [-2.5069e-02, 4.3956e-02, -4.7841e-02],
[ 3.0924e-02, 1.5174e-02, -4.8323e-02]], [ 3.0924e-02, 1.5174e-02, -4.8323e-02]],
[[-2.0325e-02, 3.2666e-02, 2.5174e-02], [[-2.0325e-02, 3.2666e-02, 2.5174e-02],
[ 5.3775e-03, -3.2712e-02, -5.2251e-02], [ 5.3775e-03, -3.2712e-02, -5.2251e-02],
[-2.7426e-02, -4.5502e-04, -3.2174e-02]], [-2.7426e-02, -4.5502e-04, -3.2174e-02]],
[[ 8.5780e-03, -5.2099e-02, 5.7285e-02], [[ 8.5780e-03, -5.2099e-02, 5.7285e-02],
[-5.0897e-02, -5.3995e-03, 4.2719e-02], [-5.0897e-02, -5.3995e-03, 4.2719e-02],
[-5.2257e-02, -7.9682e-03, 2.1848e-02]]], [-5.2257e-02, -7.9682e-03, 2.1848e-02]]],
[[[-2.8660e-02, 1.2392e-02, 2.9940e-02], [[[-2.8660e-02, 1.2392e-02, 2.9940e-02],
[-1.1170e-02, -1.7499e-02, -5.3951e-02], [-1.1170e-02, -1.7499e-02, -5.3951e-02],
[ 4.9738e-02, 2.7240e-02, 2.9588e-03]], [ 4.9738e-02, 2.7240e-02, 2.9588e-03]],
[[ 3.7728e-02, -3.1084e-02, 4.6459e-02], [[ 3.7728e-02, -3.1084e-02, 4.6459e-02],
[-1.4961e-02, 1.6951e-02, -1.3976e-02], [-1.4961e-02, 1.6951e-02, -1.3976e-02],
[ 2.5768e-02, -3.8991e-02, -2.8738e-02]], [ 2.5768e-02, -3.8991e-02, -2.8738e-02]],
[[ 1.6084e-02, -7.3174e-03, -4.5839e-02], [[ 1.6084e-02, -7.3174e-03, -4.5839e-02],
[-3.6029e-02, 1.5303e-02, -5.4380e-02], [-3.6029e-02, 1.5303e-02, -5.4380e-02],
[ 2.1913e-02, -4.4792e-02, 4.7973e-02]], [ 2.1913e-02, -4.4792e-02, 4.7973e-02]],
..., ...,
[[ 5.4945e-02, -4.5541e-02, -1.5806e-02], [[ 5.4945e-02, -4.5541e-02, -1.5806e-02],
[-4.5216e-02, -2.0338e-02, 3.3373e-02], [-4.5216e-02, -2.0338e-02, 3.3373e-02],
[-1.8431e-02, 4.3953e-02, -4.8196e-02]], [-1.8431e-02, 4.3953e-02, -4.8196e-02]],
[[-3.6129e-02, 5.7705e-02, -1.2229e-02], [[-3.6129e-02, 5.7705e-02, -1.2229e-02],
[-5.2801e-02, 7.0930e-03, -2.6721e-02], [-5.2801e-02, 7.0930e-03, -2.6721e-02],
[ 3.6720e-02, 7.5540e-04, 5.5401e-02]], [ 3.6720e-02, 7.5540e-04, 5.5401e-02]],
[[-4.0386e-02, 1.6714e-02, 2.9246e-02], [[-4.0386e-02, 1.6714e-02, 2.9246e-02],
[ 4.7033e-02, -8.7923e-03, 5.1267e-02], [ 4.7033e-02, -8.7923e-03, 5.1267e-02],
[-3.1211e-02, -2.7036e-02, 4.0346e-02]]]])), [-3.1211e-02, -2.7036e-02, 4.0346e-02]]]])),
('conv4.bias', ('conv4.bias',
tensor([-0.0368, -0.0535, -0.0568, -0.0364, 0.0167, 0.0255, 0.0031, 0.0562, tensor([-0.0368, -0.0535, -0.0568, -0.0364, 0.0167, 0.0255, 0.0031, 0.0562,
-0.0575, 0.0470, -0.0128, 0.0159, -0.0387, 0.0517, -0.0508, -0.0104, -0.0575, 0.0470, -0.0128, 0.0159, -0.0387, 0.0517, -0.0508, -0.0104,
0.0133, -0.0265, 0.0290, 0.0147, -0.0405, 0.0074, 0.0434, -0.0436, 0.0133, -0.0265, 0.0290, 0.0147, -0.0405, 0.0074, 0.0434, -0.0436,
-0.0295, 0.0417, 0.0324, 0.0316, 0.0143, -0.0312, 0.0463, 0.0441])), -0.0295, 0.0417, 0.0324, 0.0316, 0.0143, -0.0312, 0.0463, 0.0441])),
('fc1.weight', ('fc1.weight',
tensor([[-0.0206, -0.0161, -0.0053, ..., -0.0257, -0.0314, -0.0105], tensor([[-0.0206, -0.0161, -0.0053, ..., -0.0257, -0.0314, -0.0105],
[-0.0229, 0.0030, 0.0239, ..., 0.0199, -0.0279, 0.0155]])), [-0.0229, 0.0030, 0.0239, ..., 0.0199, -0.0279, 0.0155]])),
('fc1.bias', tensor([-0.5000, -1.2000]))]) ('fc1.bias', tensor([-0.5000, -1.2000]))])
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
o1.state_dict() o1.state_dict()
``` ```
%% Output %% Output
{'state': {}, {'state': {},
'param_groups': [{'lr': 0.6, 'param_groups': [{'lr': 0.6,
'momentum': 0, 'momentum': 0,
'dampening': 0, 'dampening': 0,
'weight_decay': 0, 'weight_decay': 0,
'nesterov': False, 'nesterov': False,
'params': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}]} 'params': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}]}
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
with torch.no_grad(): with torch.no_grad():
o1.param_groups[0]["params"][0].copy_(torch.zeros(1,)) o1.param_groups[0]["params"][0].copy_(torch.zeros(1,))
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
o1.param_groups o1.param_groups
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
m1.state_dict() m1.state_dict()
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
import torch import torch
loss = getattr(torch.nn.functional, 'nll_loss') loss = getattr(torch.nn.functional, 'nll_loss')
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
loss loss
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
%matplotlib inline %matplotlib inline
from decentralizepy.node.Node import Node from decentralizepy.node.Node import Node
from decentralizepy.graphs.SmallWorld import SmallWorld from decentralizepy.graphs.SmallWorld import SmallWorld
from decentralizepy.graphs.Graph import Graph from decentralizepy.graphs.Graph import Graph
from decentralizepy.mappings.Linear import Linear from decentralizepy.mappings.Linear import Linear
from torch import multiprocessing as mp from torch import multiprocessing as mp
import torch import torch
import logging import logging
from localconfig import LocalConfig from localconfig import LocalConfig
def read_ini(file_path): def read_ini(file_path):
config = LocalConfig(file_path) config = LocalConfig(file_path)
for section in config: for section in config:
print("Section: ", section) print("Section: ", section)
for key, value in config.items(section): for key, value in config.items(section):
print((key, value)) print((key, value))
print(dict(config.items('DATASET'))) print(dict(config.items('DATASET')))
return config return config
config = read_ini("config.ini") config = read_ini("config.ini")
my_config = dict() my_config = dict()
for section in config: for section in config:
my_config[section] = dict(config.items(section)) my_config[section] = dict(config.items(section))
#f = Femnist(2, 'leaf/data/femnist/data/train', sizes=[0.6, 0.4]) #f = Femnist(2, 'leaf/data/femnist/data/train', sizes=[0.6, 0.4])
g = Graph() g = Graph()
g.read_graph_from_file("36_nodes.edges", "edges") g.read_graph_from_file("36_nodes.edges", "edges")
l = Linear(1, 36) l = Linear(1, 36)
#Node(0, 0, l, g, my_config, 20, "results", logging.DEBUG) #Node(0, 0, l, g, my_config, 20, "results", logging.DEBUG)
mp.spawn(fn = Node, nprocs = g.n_procs, args=[0,l,g,my_config,20,"results",logging.INFO]) mp.spawn(fn = Node, nprocs = g.n_procs, args=[0,l,g,my_config,20,"results",logging.INFO])
# mp.spawn(fn = Node, args = [l, g, config, 10, "results", logging.DEBUG], nprocs=2) # mp.spawn(fn = Node, args = [l, g, config, 10, "results", logging.DEBUG], nprocs=2)
``` ```
%% Output %% Output
Section: GRAPH Section: GRAPH
('package', 'decentralizepy.graphs.SmallWorld') ('package', 'decentralizepy.graphs.SmallWorld')
('graph_class', 'SmallWorld') ('graph_class', 'SmallWorld')
Section: DATASET Section: DATASET
('dataset_package', 'decentralizepy.datasets.Femnist') ('dataset_package', 'decentralizepy.datasets.Femnist')
('dataset_class', 'Femnist') ('dataset_class', 'Femnist')
('model_class', 'CNN') ('model_class', 'CNN')
('n_procs', 36) ('n_procs', 36)
('train_dir', 'leaf/data/femnist/per_user_data/train') ('train_dir', 'leaf/data/femnist/per_user_data/train')
('test_dir', 'leaf/data/femnist/data/test') ('test_dir', 'leaf/data/femnist/data/test')
('sizes', '') ('sizes', '')
Section: OPTIMIZER_PARAMS Section: OPTIMIZER_PARAMS
('optimizer_package', 'torch.optim') ('optimizer_package', 'torch.optim')
('optimizer_class', 'Adam') ('optimizer_class', 'Adam')
('lr', 0.01) ('lr', 0.01)
Section: TRAIN_PARAMS Section: TRAIN_PARAMS
('training_package', 'decentralizepy.training.Training') ('training_package', 'decentralizepy.training.Training')
('training_class', 'Training') ('training_class', 'Training')
('epochs_per_round', 1) ('epochs_per_round', 1)
('batch_size', 1024) ('batch_size', 1024)
('shuffle', True) ('shuffle', True)
('loss_package', 'torch.nn') ('loss_package', 'torch.nn')
('loss_class', 'CrossEntropyLoss') ('loss_class', 'CrossEntropyLoss')
Section: COMMUNICATION Section: COMMUNICATION
('comm_package', 'decentralizepy.communication.TCP') ('comm_package', 'decentralizepy.communication.TCP')
('comm_class', 'TCP') ('comm_class', 'TCP')
('addresses_filepath', 'ip_addr.json') ('addresses_filepath', 'ip_addr.json')
Section: SHARING Section: SHARING
('sharing_package', 'decentralizepy.sharing.Sharing') ('sharing_package', 'decentralizepy.sharing.Sharing')
('sharing_class', 'Sharing') ('sharing_class', 'Sharing')
{'dataset_package': 'decentralizepy.datasets.Femnist', 'dataset_class': 'Femnist', 'model_class': 'CNN', 'n_procs': 36, 'train_dir': 'leaf/data/femnist/per_user_data/train', 'test_dir': 'leaf/data/femnist/data/test', 'sizes': ''} {'dataset_package': 'decentralizepy.datasets.Femnist', 'dataset_class': 'Femnist', 'model_class': 'CNN', 'n_procs': 36, 'train_dir': 'leaf/data/femnist/per_user_data/train', 'test_dir': 'leaf/data/femnist/data/test', 'sizes': ''}
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
from decentralizepy.mappings.Linear import Linear from decentralizepy.mappings.Linear import Linear
from testing import f from testing import f
from torch import multiprocessing as mp from torch import multiprocessing as mp
l = Linear(1, 2) l = Linear(1, 2)
mp.spawn(fn = f, nprocs = 2, args = [0, 2, "ip_addr.json", l]) mp.spawn(fn = f, nprocs = 2, args = [0, 2, "ip_addr.json", l])
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
from decentralizepy.datasets.Celeba import Celeba from decentralizepy.datasets.Celeba import Celeba
f = Celeba(0,0,None) f = Celeba(0,0,None)
f.file_per_user('/home/risharma/leaf/data/celeba/data/train','/home/risharma/leaf/data/celeba/per_user_data/train') f.file_per_user('/home/risharma/leaf/data/celeba/data/train','/home/risharma/leaf/data/celeba/per_user_data/train')
``` ```
%% Output %% Output
Created File: 3020.json Created File: 3020.json
Created File: 6675.json Created File: 6675.json
Created File: 6725.json Created File: 6725.json
Created File: 2263.json Created File: 2263.json
Created File: 1321.json Created File: 1321.json
Created File: 8721.json Created File: 8721.json
Created File: 8722.json Created File: 8722.json
Created File: 8822.json Created File: 8822.json
Created File: 1448.json Created File: 1448.json
Created File: 6568.json Created File: 6568.json
Created File: 7439.json Created File: 7439.json
Created File: 4905.json Created File: 4905.json
Created File: 4153.json Created File: 4153.json
Created File: 8860.json Created File: 8860.json
Created File: 3941.json Created File: 3941.json
Created File: 4147.json Created File: 4147.json
Created File: 6398.json Created File: 6398.json
Created File: 4030.json Created File: 4030.json
Created File: 1050.json Created File: 1050.json
Created File: 7261.json Created File: 7261.json
Created File: 4602.json Created File: 4602.json
Created File: 8910.json Created File: 8910.json
Created File: 7165.json Created File: 7165.json
Created File: 3508.json Created File: 3508.json
Created File: 5620.json Created File: 5620.json
Created File: 6648.json Created File: 6648.json
Created File: 9790.json Created File: 9790.json
Created File: 8630.json Created File: 8630.json
Created File: 9713.json Created File: 9713.json
Created File: 3751.json Created File: 3751.json
Created File: 3312.json Created File: 3312.json
Created File: 4773.json Created File: 4773.json
Created File: 4582.json Created File: 4582.json
Created File: 9798.json Created File: 9798.json
Created File: 2897.json Created File: 2897.json
Created File: 2859.json Created File: 2859.json
Created File: 562.json Created File: 562.json
Created File: 6334.json Created File: 6334.json
Created File: 3055.json Created File: 3055.json
Created File: 7932.json Created File: 7932.json
Created File: 1218.json Created File: 1218.json
Created File: 5157.json Created File: 5157.json
Created File: 1151.json Created File: 1151.json
Created File: 6300.json Created File: 6300.json
Created File: 8916.json Created File: 8916.json
Created File: 5768.json Created File: 5768.json
Created File: 9821.json Created File: 9821.json
Created File: 8560.json Created File: 8560.json
Created File: 9984.json Created File: 9984.json
Created File: 2831.json Created File: 2831.json
Created File: 5193.json Created File: 5193.json
Created File: 5457.json Created File: 5457.json
Created File: 4647.json Created File: 4647.json
Created File: 2861.json Created File: 2861.json
Created File: 6135.json Created File: 6135.json
Created File: 6429.json Created File: 6429.json
Created File: 4860.json Created File: 4860.json
Created File: 2837.json Created File: 2837.json
Created File: 7206.json Created File: 7206.json
Created File: 3325.json Created File: 3325.json
Created File: 8962.json Created File: 8962.json
Created File: 4395.json Created File: 4395.json
Created File: 4601.json Created File: 4601.json
Created File: 8826.json Created File: 8826.json
Created File: 1370.json Created File: 1370.json
Created File: 7760.json Created File: 7760.json
Created File: 6152.json Created File: 6152.json
Created File: 3410.json Created File: 3410.json
Created File: 4094.json Created File: 4094.json
Created File: 4458.json Created File: 4458.json
Created File: 6555.json Created File: 6555.json
Created File: 8787.json Created File: 8787.json
Created File: 5326.json Created File: 5326.json
Created File: 5138.json Created File: 5138.json
Created File: 5173.json Created File: 5173.json
Created File: 9946.json Created File: 9946.json
Created File: 2547.json Created File: 2547.json
Created File: 5386.json Created File: 5386.json
Created File: 4715.json Created File: 4715.json
Created File: 10083.json Created File: 10083.json
Created File: 3027.json Created File: 3027.json
Created File: 78.json Created File: 78.json
Created File: 6128.json Created File: 6128.json
Created File: 5512.json Created File: 5512.json
Created File: 5093.json Created File: 5093.json
Created File: 2334.json Created File: 2334.json
Created File: 6422.json Created File: 6422.json
Created File: 6346.json Created File: 6346.json
Created File: 4312.json Created File: 4312.json
Created File: 1346.json Created File: 1346.json
Created File: 5131.json Created File: 5131.json
Created File: 2374.json Created File: 2374.json
Created File: 5340.json Created File: 5340.json
Created File: 9965.json Created File: 9965.json
Created File: 5310.json Created File: 5310.json
Created File: 7245.json Created File: 7245.json
Created File: 3278.json Created File: 3278.json
Created File: 6415.json Created File: 6415.json
Created File: 3614.json Created File: 3614.json
Created File: 801.json Created File: 801.json
Created File: 5495.json Created File: 5495.json
Created File: 318.json Created File: 318.json
Created File: 6705.json Created File: 6705.json
Created File: 3398.json Created File: 3398.json
Created File: 8641.json Created File: 8641.json
Created File: 7233.json Created File: 7233.json
Created File: 9341.json Created File: 9341.json
Created File: 1994.json Created File: 1994.json
Created File: 6137.json Created File: 6137.json
Created File: 6191.json Created File: 6191.json
Created File: 5007.json Created File: 5007.json
Created File: 10025.json Created File: 10025.json
Created File: 2055.json Created File: 2055.json
Created File: 4410.json Created File: 4410.json
Created File: 8148.json Created File: 8148.json
Created File: 5008.json Created File: 5008.json
Created File: 5042.json Created File: 5042.json
Created File: 4262.json Created File: 4262.json
Created File: 3034.json Created File: 3034.json
Created File: 2193.json Created File: 2193.json
Created File: 4076.json Created File: 4076.json
Created File: 1181.json Created File: 1181.json
Created File: 5128.json Created File: 5128.json
Created File: 7768.json Created File: 7768.json
Created File: 844.json Created File: 844.json
Created File: 8677.json Created File: 8677.json
Created File: 8386.json Created File: 8386.json
Created File: 10175.json Created File: 10175.json
Created File: 7186.json Created File: 7186.json
Created File: 3864.json Created File: 3864.json
Created File: 10032.json Created File: 10032.json
Created File: 1903.json Created File: 1903.json
Created File: 3037.json Created File: 3037.json
Created File: 3397.json Created File: 3397.json
Created File: 4863.json Created File: 4863.json
Created File: 4539.json Created File: 4539.json
Created File: 9010.json Created File: 9010.json
Created File: 4472.json Created File: 4472.json
Created File: 5688.json Created File: 5688.json
Created File: 1625.json Created File: 1625.json
Created File: 6165.json Created File: 6165.json
Created File: 7680.json Created File: 7680.json
Created File: 8469.json Created File: 8469.json
Created File: 6936.json Created File: 6936.json
Created File: 8341.json Created File: 8341.json
Created File: 1034.json Created File: 1034.json
Created File: 6267.json Created File: 6267.json
Created File: 4435.json Created File: 4435.json
Created File: 8831.json Created File: 8831.json
Created File: 22.json Created File: 22.json
Created File: 2960.json Created File: 2960.json
Created File: 1059.json Created File: 1059.json
Created File: 3040.json Created File: 3040.json
Created File: 7192.json Created File: 7192.json
Created File: 5563.json Created File: 5563.json
Created File: 8788.json Created File: 8788.json
Created File: 3329.json Created File: 3329.json
Created File: 6411.json Created File: 6411.json
Created File: 5151.json Created File: 5151.json
Created File: 5321.json Created File: 5321.json
Created File: 4328.json Created File: 4328.json
Created File: 7516.json Created File: 7516.json
Created File: 2899.json Created File: 2899.json
Created File: 8586.json Created File: 8586.json
Created File: 8933.json Created File: 8933.json
Created File: 1435.json Created File: 1435.json
Created File: 8803.json Created File: 8803.json
Created File: 3436.json Created File: 3436.json
Created File: 7222.json Created File: 7222.json
Created File: 1221.json Created File: 1221.json
Created File: 4691.json Created File: 4691.json
Created File: 2332.json Created File: 2332.json
Created File: 2125.json Created File: 2125.json
Created File: 1755.json Created File: 1755.json
Created File: 8850.json Created File: 8850.json
Created File: 6232.json Created File: 6232.json
Created File: 6127.json Created File: 6127.json
Created File: 3203.json Created File: 3203.json
Created File: 5092.json Created File: 5092.json
Created File: 4110.json Created File: 4110.json
Created File: 2524.json Created File: 2524.json
Created File: 8672.json Created File: 8672.json
Created File: 7519.json Created File: 7519.json
Created File: 2309.json Created File: 2309.json
Created File: 3059.json Created File: 3059.json
Created File: 10073.json Created File: 10073.json
Created File: 7237.json Created File: 7237.json
Created File: 8706.json Created File: 8706.json
Created File: 5471.json Created File: 5471.json
Created File: 5252.json Created File: 5252.json
Created File: 7335.json Created File: 7335.json
Created File: 7450.json Created File: 7450.json
Created File: 1188.json Created File: 1188.json
Created File: 5794.json Created File: 5794.json
Created File: 2758.json Created File: 2758.json
Created File: 6046.json Created File: 6046.json
Created File: 1927.json Created File: 1927.json
Created File: 2014.json Created File: 2014.json
Created File: 2880.json Created File: 2880.json
Created File: 3930.json Created File: 3930.json
Created File: 10070.json Created File: 10070.json
Created File: 6316.json Created File: 6316.json
Created File: 7611.json Created File: 7611.json
Created File: 2445.json Created File: 2445.json
Created File: 7087.json Created File: 7087.json
Created File: 3080.json Created File: 3080.json
Created File: 5418.json Created File: 5418.json
Created File: 1175.json Created File: 1175.json
Created File: 7596.json Created File: 7596.json
Created File: 2791.json Created File: 2791.json
Created File: 3029.json Created File: 3029.json
Created File: 5147.json Created File: 5147.json
Created File: 8482.json Created File: 8482.json
Created File: 1524.json Created File: 1524.json
Created File: 8875.json Created File: 8875.json
Created File: 9401.json Created File: 9401.json
Created File: 1033.json Created File: 1033.json
Created File: 2083.json Created File: 2083.json
Created File: 4991.json Created File: 4991.json
Created File: 2914.json Created File: 2914.json
Created File: 3248.json Created File: 3248.json
Created File: 4625.json Created File: 4625.json
Created File: 6864.json Created File: 6864.json
Created File: 1272.json Created File: 1272.json
Created File: 10156.json Created File: 10156.json
Created File: 8547.json Created File: 8547.json
Created File: 659.json Created File: 659.json
Created File: 1031.json Created File: 1031.json
Created File: 4447.json Created File: 4447.json
Created File: 4911.json Created File: 4911.json
Created File: 9179.json Created File: 9179.json
Created File: 661.json Created File: 661.json
Created File: 8627.json Created File: 8627.json
Created File: 8072.json Created File: 8072.json
Created File: 1373.json Created File: 1373.json
Created File: 2863.json Created File: 2863.json
Created File: 1468.json Created File: 1468.json
Created File: 9056.json Created File: 9056.json
Created File: 8498.json Created File: 8498.json
Created File: 9020.json Created File: 9020.json
Created File: 8355.json Created File: 8355.json
Created File: 7126.json Created File: 7126.json
Created File: 2433.json Created File: 2433.json
Created File: 7228.json Created File: 7228.json
Created File: 6802.json Created File: 6802.json
Created File: 6708.json Created File: 6708.json
Created File: 3014.json Created File: 3014.json
Created File: 4138.json Created File: 4138.json
Created File: 9246.json Created File: 9246.json
Created File: 7399.json Created File: 7399.json
Created File: 6320.json Created File: 6320.json
Created File: 5350.json Created File: 5350.json
Created File: 9758.json Created File: 9758.json
Created File: 6355.json Created File: 6355.json
Created File: 8471.json Created File: 8471.json
Created File: 6036.json Created File: 6036.json
Created File: 2937.json Created File: 2937.json
Created File: 7001.json Created File: 7001.json
Created File: 7367.json Created File: 7367.json
Created File: 492.json Created File: 492.json
Created File: 4242.json Created File: 4242.json
Created File: 4157.json Created File: 4157.json
Created File: 5715.json Created File: 5715.json
Created File: 7488.json Created File: 7488.json
Created File: 6343.json Created File: 6343.json
Created File: 6536.json Created File: 6536.json
Created File: 9986.json Created File: 9986.json
Created File: 5246.json Created File: 5246.json
Created File: 3449.json Created File: 3449.json
Created File: 9843.json Created File: 9843.json
Created File: 2533.json Created File: 2533.json
Created File: 6100.json Created File: 6100.json
Created File: 7613.json Created File: 7613.json
Created File: 4310.json Created File: 4310.json
Created File: 3304.json Created File: 3304.json
Created File: 1985.json Created File: 1985.json
Created File: 9987.json Created File: 9987.json
Created File: 47.json Created File: 47.json
Created File: 2983.json Created File: 2983.json
Created File: 9722.json Created File: 9722.json
Created File: 7315.json Created File: 7315.json
Created File: 3782.json Created File: 3782.json
Created File: 9918.json Created File: 9918.json
Created File: 1569.json Created File: 1569.json
Created File: 5035.json Created File: 5035.json
Created File: 5101.json Created File: 5101.json
Created File: 5567.json Created File: 5567.json
Created File: 6181.json Created File: 6181.json
Created File: 8966.json Created File: 8966.json
Created File: 3377.json Created File: 3377.json
Created File: 1580.json Created File: 1580.json
Created File: 10134.json Created File: 10134.json
Created File: 1003.json Created File: 1003.json
Created File: 7286.json Created File: 7286.json
Created File: 6567.json Created File: 6567.json
Created File: 1726.json Created File: 1726.json
Created File: 992.json Created File: 992.json
Created File: 1809.json Created File: 1809.json
Created File: 8693.json Created File: 8693.json
Created File: 4561.json Created File: 4561.json
Created File: 6385.json Created File: 6385.json
Created File: 4168.json Created File: 4168.json
Created File: 4149.json Created File: 4149.json
Created File: 2070.json Created File: 2070.json
Created File: 4165.json Created File: 4165.json
Created File: 9701.json Created File: 9701.json
Created File: 4162.json Created File: 4162.json
Created File: 3620.json Created File: 3620.json
Created File: 7764.json Created File: 7764.json
Created File: 9014.json Created File: 9014.json
Created File: 7178.json Created File: 7178.json
Created File: 6043.json Created File: 6043.json
Created File: 2236.json Created File: 2236.json
Created File: 1544.json Created File: 1544.json
Created File: 1744.json Created File: 1744.json
Created File: 5613.json Created File: 5613.json
Created File: 8485.json Created File: 8485.json
Created File: 8578.json Created File: 8578.json
Created File: 4516.json Created File: 4516.json
Created File: 6995.json Created File: 6995.json
Created File: 4126.json Created File: 4126.json
Created File: 8774.json Created File: 8774.json
Created File: 8164.json Created File: 8164.json
Created File: 8638.json Created File: 8638.json
Created File: 2599.json Created File: 2599.json
Created File: 5158.json Created File: 5158.json
Created File: 3670.json Created File: 3670.json
Created File: 2221.json Created File: 2221.json
Created File: 8868.json Created File: 8868.json
Created File: 4190.json Created File: 4190.json
Created File: 2054.json Created File: 2054.json
Created File: 4070.json Created File: 4070.json
Created File: 6823.json Created File: 6823.json
Created File: 5549.json Created File: 5549.json
Created File: 334.json Created File: 334.json
Created File: 5569.json Created File: 5569.json
Created File: 5086.json Created File: 5086.json
Created File: 1574.json Created File: 1574.json
Created File: 2243.json Created File: 2243.json
Created File: 7358.json Created File: 7358.json
Created File: 4068.json Created File: 4068.json
Created File: 2429.json Created File: 2429.json
Created File: 6134.json Created File: 6134.json
Created File: 4114.json Created File: 4114.json
Created File: 6614.json Created File: 6614.json
Created File: 6325.json Created File: 6325.json
Created File: 3808.json Created File: 3808.json
Created File: 2078.json Created File: 2078.json
Created File: 7424.json Created File: 7424.json
Created File: 2523.json Created File: 2523.json
Created File: 8685.json Created File: 8685.json
Created File: 5223.json Created File: 5223.json
Created File: 409.json Created File: 409.json
Created File: 3185.json Created File: 3185.json
Created File: 5581.json Created File: 5581.json
Created File: 2972.json Created File: 2972.json
Created File: 2072.json Created File: 2072.json
Created File: 4172.json Created File: 4172.json
Created File: 9123.json Created File: 9123.json
Created File: 6546.json Created File: 6546.json
Created File: 3912.json Created File: 3912.json
Created File: 8951.json Created File: 8951.json
Created File: 2819.json Created File: 2819.json
Created File: 510.json Created File: 510.json
Created File: 6583.json Created File: 6583.json
Created File: 2314.json Created File: 2314.json
Created File: 8747.json Created File: 8747.json
Created File: 7569.json Created File: 7569.json
Created File: 3024.json Created File: 3024.json
Created File: 5198.json Created File: 5198.json
Created File: 9669.json Created File: 9669.json
Created File: 7156.json Created File: 7156.json
Created File: 5126.json Created File: 5126.json
Created File: 6314.json Created File: 6314.json
Created File: 8945.json Created File: 8945.json
Created File: 4946.json Created File: 4946.json
Created File: 8098.json Created File: 8098.json
Created File: 8528.json Created File: 8528.json
Created File: 7571.json Created File: 7571.json
Created File: 1314.json Created File: 1314.json
Created File: 8942.json Created File: 8942.json
Created File: 9719.json Created File: 9719.json
Created File: 2240.json Created File: 2240.json
Created File: 9011.json Created File: 9011.json
Created File: 7548.json Created File: 7548.json
Created File: 2155.json Created File: 2155.json
Created File: 9158.json Created File: 9158.json
Created File: 1912.json Created File: 1912.json
Created File: 481.json Created File: 481.json
Created File: 4255.json Created File: 4255.json
Created File: 8704.json Created File: 8704.json
Created File: 9944.json Created File: 9944.json
Created File: 4825.json Created File: 4825.json
Created File: 2957.json Created File: 2957.json
Created File: 3768.json Created File: 3768.json
Created File: 4884.json Created File: 4884.json
Created File: 248.json Created File: 248.json
Created File: 1999.json Created File: 1999.json
Created File: 8643.json Created File: 8643.json
Created File: 3140.json Created File: 3140.json
Created File: 2136.json Created File: 2136.json
Created File: 7409.json Created File: 7409.json
Created File: 1499.json Created File: 1499.json
Created File: 9610.json Created File: 9610.json
Created File: 5148.json Created File: 5148.json
Created File: 7845.json Created File: 7845.json
Created File: 2026.json Created File: 2026.json
Created File: 1588.json Created File: 1588.json
Created File: 4571.json Created File: 4571.json
Created File: 4290.json Created File: 4290.json
Created File: 2101.json Created File: 2101.json
Created File: 5507.json Created File: 5507.json
Created File: 9850.json Created File: 9850.json
Created File: 147.json Created File: 147.json
Created File: 8208.json Created File: 8208.json
Created File: 7271.json Created File: 7271.json
Created File: 6809.json Created File: 6809.json
Created File: 6483.json Created File: 6483.json
Created File: 10003.json Created File: 10003.json
Created File: 5124.json Created File: 5124.json
Created File: 5427.json Created File: 5427.json
Created File: 69.json Created File: 69.json
Created File: 3928.json Created File: 3928.json
Created File: 1601.json Created File: 1601.json
Created File: 4808.json Created File: 4808.json
Created File: 8667.json Created File: 8667.json
Created File: 4473.json Created File: 4473.json
Created File: 3198.json Created File: 3198.json
Created File: 2612.json Created File: 2612.json
Created File: 6677.json Created File: 6677.json
Created File: 5250.json Created File: 5250.json
Created File: 6301.json Created File: 6301.json
Created File: 5702.json Created File: 5702.json
Created File: 8501.json Created File: 8501.json
Created File: 5452.json Created File: 5452.json
Created File: 4828.json Created File: 4828.json
Created File: 9116.json Created File: 9116.json
Created File: 4028.json Created File: 4028.json
Created File: 3845.json Created File: 3845.json
Created File: 2790.json Created File: 2790.json
Created File: 1689.json Created File: 1689.json
Created File: 1986.json Created File: 1986.json
Created File: 8563.json Created File: 8563.json
Created File: 467.json Created File: 467.json
Created File: 1302.json Created File: 1302.json
Created File: 4438.json Created File: 4438.json
Created File: 1784.json Created File: 1784.json
Created File: 7755.json Created File: 7755.json
Created File: 5178.json Created File: 5178.json
Created File: 4252.json Created File: 4252.json
Created File: 10080.json Created File: 10080.json
Created File: 10085.json Created File: 10085.json
Created File: 3875.json Created File: 3875.json
Created File: 4944.json Created File: 4944.json
Created File: 9152.json Created File: 9152.json
Created File: 5202.json Created File: 5202.json
Created File: 6185.json Created File: 6185.json
Created File: 1885.json Created File: 1885.json
Created File: 8521.json Created File: 8521.json
Created File: 8500.json Created File: 8500.json
Created File: 8379.json Created File: 8379.json
Created File: 6688.json Created File: 6688.json
Created File: 8275.json Created File: 8275.json
Created File: 981.json Created File: 981.json
Created File: 8702.json Created File: 8702.json
Created File: 6997.json Created File: 6997.json
Created File: 7607.json Created File: 7607.json
Created File: 9166.json Created File: 9166.json
Created File: 2020.json Created File: 2020.json
Created File: 1660.json Created File: 1660.json
Created File: 9807.json Created File: 9807.json
Created File: 6633.json Created File: 6633.json
Created File: 1651.json Created File: 1651.json
Created File: 5551.json Created File: 5551.json
Created File: 2792.json Created File: 2792.json
Created File: 6365.json Created File: 6365.json
Created File: 4446.json Created File: 4446.json
Created File: 3501.json Created File: 3501.json
Created File: 1537.json Created File: 1537.json
Created File: 2507.json Created File: 2507.json
Created File: 6200.json Created File: 6200.json
Created File: 6412.json Created File: 6412.json
Created File: 988.json Created File: 988.json
Created File: 1713.json Created File: 1713.json
Created File: 496.json Created File: 496.json
Created File: 9864.json Created File: 9864.json
Created File: 6081.json Created File: 6081.json
Created File: 8825.json Created File: 8825.json
Created File: 6565.json Created File: 6565.json
Created File: 4955.json Created File: 4955.json
Created File: 8709.json Created File: 8709.json
Created File: 4865.json Created File: 4865.json
Created File: 8810.json Created File: 8810.json
Created File: 7449.json Created File: 7449.json
Created File: 199.json Created File: 199.json
Created File: 7475.json Created File: 7475.json
Created File: 2969.json Created File: 2969.json
Created File: 4288.json Created File: 4288.json
Created File: 8786.json Created File: 8786.json
Created File: 5273.json Created File: 5273.json
Created File: 8195.json Created File: 8195.json
Created File: 6166.json Created File: 6166.json
Created File: 9810.json Created File: 9810.json
Created File: 2095.json Created File: 2095.json
Created File: 4141.json Created File: 4141.json
Created File: 1224.json Created File: 1224.json
Created File: 2124.json Created File: 2124.json
Created File: 3051.json Created File: 3051.json
Created File: 7095.json Created File: 7095.json
Created File: 8847.json Created File: 8847.json
Created File: 7923.json Created File: 7923.json
Created File: 1614.json Created File: 1614.json
Created File: 5953.json Created File: 5953.json
Created File: 7867.json Created File: 7867.json
Created File: 9128.json Created File: 9128.json
Created File: 8943.json Created File: 8943.json
Created File: 7673.json Created File: 7673.json
Created File: 8807.json Created File: 8807.json
Created File: 5133.json Created File: 5133.json
Created File: 8420.json Created File: 8420.json
Created File: 9031.json Created File: 9031.json
Created File: 3882.json Created File: 3882.json
Created File: 3681.json Created File: 3681.json
Created File: 9874.json Created File: 9874.json
Created File: 3714.json Created File: 3714.json
Created File: 4537.json Created File: 4537.json
Created File: 3407.json Created File: 3407.json
Created File: 3404.json Created File: 3404.json
Created File: 6024.json Created File: 6024.json
Created File: 4874.json Created File: 4874.json
Created File: 6920.json Created File: 6920.json
Created File: 918.json Created File: 918.json
Created File: 3383.json Created File: 3383.json
Created File: 4987.json Created File: 4987.json
Created File: 1341.json Created File: 1341.json
Created File: 3270.json Created File: 3270.json
Created File: 5028.json Created File: 5028.json
Created File: 8727.json Created File: 8727.json
Created File: 4819.json Created File: 4819.json
Created File: 9698.json Created File: 9698.json
Created File: 1916.json Created File: 1916.json
Created File: 4923.json Created File: 4923.json
Created File: 2030.json Created File: 2030.json
Created File: 4805.json Created File: 4805.json
Created File: 2820.json Created File: 2820.json
Created File: 3084.json Created File: 3084.json
Created File: 6291.json Created File: 6291.json
Created File: 94.json Created File: 94.json
Created File: 2160.json Created File: 2160.json
Created File: 1589.json Created File: 1589.json
Created File: 4543.json Created File: 4543.json
Created File: 1664.json Created File: 1664.json
Created File: 5382.json Created File: 5382.json
Created File: 4421.json Created File: 4421.json
Created File: 2602.json Created File: 2602.json
Created File: 5370.json Created File: 5370.json
Created File: 397.json Created File: 397.json
Created File: 6703.json Created File: 6703.json
Created File: 4119.json Created File: 4119.json
Created File: 1037.json Created File: 1037.json
Created File: 5799.json Created File: 5799.json
Created File: 4801.json Created File: 4801.json
Created File: 4244.json Created File: 4244.json
Created File: 1731.json Created File: 1731.json
Created File: 5638.json Created File: 5638.json
Created File: 6628.json Created File: 6628.json
Created File: 4014.json Created File: 4014.json
Created File: 8385.json Created File: 8385.json
Created File: 8558.json Created File: 8558.json
Created File: 3696.json Created File: 3696.json
Created File: 2195.json Created File: 2195.json
Created File: 6637.json Created File: 6637.json
Created File: 3432.json Created File: 3432.json
Created File: 9818.json Created File: 9818.json
Created File: 9794.json Created File: 9794.json
Created File: 1335.json Created File: 1335.json
Created File: 8440.json Created File: 8440.json
Created File: 6988.json Created File: 6988.json
Created File: 3496.json Created File: 3496.json
Created File: 5096.json Created File: 5096.json
Created File: 7524.json Created File: 7524.json
Created File: 8365.json Created File: 8365.json
Created File: 4316.json Created File: 4316.json
Created File: 4031.json Created File: 4031.json
Created File: 8928.json Created File: 8928.json
Created File: 7309.json Created File: 7309.json
Created File: 2120.json Created File: 2120.json
Created File: 1686.json Created File: 1686.json
Created File: 565.json Created File: 565.json
Created File: 7282.json Created File: 7282.json
Created File: 977.json Created File: 977.json
Created File: 3113.json Created File: 3113.json
Created File: 2884.json Created File: 2884.json
Created File: 3098.json Created File: 3098.json
Created File: 449.json Created File: 449.json
Created File: 4816.json Created File: 4816.json
Created File: 6680.json Created File: 6680.json
Created File: 5132.json Created File: 5132.json
Created File: 7074.json Created File: 7074.json
Created File: 991.json Created File: 991.json
Created File: 114.json Created File: 114.json
Created File: 6596.json Created File: 6596.json
Created File: 4671.json Created File: 4671.json
Created File: 3546.json Created File: 3546.json
Created File: 4747.json Created File: 4747.json
Created File: 6190.json Created File: 6190.json
Created File: 9808.json Created File: 9808.json
Created File: 3661.json Created File: 3661.json
Created File: 447.json Created File: 447.json
Created File: 3338.json Created File: 3338.json
Created File: 3129.json Created File: 3129.json
Created File: 4061.json Created File: 4061.json
Created File: 5073.json Created File: 5073.json
Created File: 973.json Created File: 973.json
Created File: 2261.json Created File: 2261.json
Created File: 1140.json Created File: 1140.json
Created File: 9682.json Created File: 9682.json
Created File: 6059.json Created File: 6059.json
Created File: 6502.json Created File: 6502.json
Created File: 4577.json Created File: 4577.json
Created File: 7533.json Created File: 7533.json
Created File: 2201.json Created File: 2201.json
Created File: 7083.json Created File: 7083.json
Created File: 7408.json Created File: 7408.json
Created File: 1018.json Created File: 1018.json
Created File: 5609.json Created File: 5609.json
Created File: 5341.json Created File: 5341.json
Created File: 9213.json Created File: 9213.json
Created File: 2084.json Created File: 2084.json
Created File: 8419.json Created File: 8419.json
Created File: 1090.json Created File: 1090.json
Created File: 1061.json Created File: 1061.json
Created File: 4419.json Created File: 4419.json
Created File: 9768.json Created File: 9768.json
Created File: 5293.json Created File: 5293.json
Created File: 5391.json Created File: 5391.json
Created File: 9878.json Created File: 9878.json
Created File: 305.json Created File: 305.json
Created File: 10159.json Created File: 10159.json
Created File: 8861.json Created File: 8861.json
Created File: 2562.json Created File: 2562.json
Created File: 21.json Created File: 21.json
Created File: 6683.json Created File: 6683.json
Created File: 2063.json Created File: 2063.json
Created File: 8121.json Created File: 8121.json
Created File: 1264.json Created File: 1264.json
Created File: 7317.json Created File: 7317.json
Created File: 8609.json Created File: 8609.json
Created File: 4315.json Created File: 4315.json
Created File: 614.json Created File: 614.json
Created File: 1051.json Created File: 1051.json
Created File: 3386.json Created File: 3386.json
Created File: 8514.json Created File: 8514.json
Created File: 4814.json Created File: 4814.json
Created File: 6696.json Created File: 6696.json
Created File: 6262.json Created File: 6262.json
Created File: 2516.json Created File: 2516.json
Created File: 7567.json Created File: 7567.json
Created File: 1014.json Created File: 1014.json
Created File: 6702.json Created File: 6702.json
Created File: 3134.json Created File: 3134.json
Created File: 5473.json Created File: 5473.json
Created File: 4308.json Created File: 4308.json
Created File: 1394.json Created File: 1394.json
Created File: 735.json Created File: 735.json
Created File: 8936.json Created File: 8936.json
Created File: 10062.json Created File: 10062.json
Created File: 1209.json Created File: 1209.json
Created File: 9935.json Created File: 9935.json
Created File: 9155.json Created File: 9155.json
Created File: 2926.json Created File: 2926.json
Created File: 2311.json Created File: 2311.json
Created File: 2509.json Created File: 2509.json
Created File: 1714.json Created File: 1714.json
Created File: 10167.json Created File: 10167.json
Created File: 4974.json Created File: 4974.json
Created File: 4027.json Created File: 4027.json
Created File: 4177.json Created File: 4177.json
Created File: 3118.json Created File: 3118.json
Created File: 6994.json Created File: 6994.json
Created File: 8313.json Created File: 8313.json
Created File: 1711.json Created File: 1711.json
Created File: 6318.json Created File: 6318.json
Created File: 6811.json Created File: 6811.json
Created File: 958.json Created File: 958.json
Created File: 1984.json Created File: 1984.json
Created File: 2472.json Created File: 2472.json
Created File: 2883.json Created File: 2883.json
Created File: 2573.json Created File: 2573.json
Created File: 8644.json Created File: 8644.json
Created File: 4723.json Created File: 4723.json
Created File: 8944.json Created File: 8944.json
Created File: 5672.json Created File: 5672.json
Created File: 894.json Created File: 894.json
Created File: 5238.json Created File: 5238.json
Created File: 5897.json Created File: 5897.json
Created File: 4055.json Created File: 4055.json
Created File: 5928.json Created File: 5928.json
Created File: 8011.json Created File: 8011.json
Created File: 6182.json Created File: 6182.json
Created File: 4719.json Created File: 4719.json
Created File: 1011.json Created File: 1011.json
Created File: 8771.json Created File: 8771.json
Created File: 1154.json Created File: 1154.json
Created File: 8494.json Created File: 8494.json
Created File: 4125.json Created File: 4125.json
Created File: 1854.json Created File: 1854.json
Created File: 4453.json Created File: 4453.json
Created File: 8495.json Created File: 8495.json
Created File: 2373.json Created File: 2373.json
Created File: 5491.json Created File: 5491.json
Created File: 6877.json Created File: 6877.json
Created File: 4487.json Created File: 4487.json
Created File: 6986.json Created File: 6986.json
Created File: 10016.json Created File: 10016.json
Created File: 4744.json Created File: 4744.json
Created File: 3619.json Created File: 3619.json
Created File: 7346.json Created File: 7346.json
Created File: 1581.json Created File: 1581.json
Created File: 4436.json Created File: 4436.json
Created File: 8736.json Created File: 8736.json
Created File: 3725.json Created File: 3725.json
Created File: 8954.json Created File: 8954.json
Created File: 3697.json Created File: 3697.json
Created File: 1700.json Created File: 1700.json
Created File: 2249.json Created File: 2249.json
Created File: 3915.json Created File: 3915.json
Created File: 9927.json Created File: 9927.json
Created File: 7392.json Created File: 7392.json
Created File: 1603.json Created File: 1603.json
Created File: 10174.json Created File: 10174.json
Created File: 1054.json Created File: 1054.json
Created File: 6219.json Created File: 6219.json
Created File: 4853.json Created File: 4853.json
Created File: 1604.json Created File: 1604.json
Created File: 1990.json Created File: 1990.json
Created File: 5185.json Created File: 5185.json
Created File: 4462.json Created File: 4462.json
Created File: 9006.json Created File: 9006.json
Created File: 3896.json Created File: 3896.json
Created File: 1899.json Created File: 1899.json
Created File: 7362.json Created File: 7362.json
Created File: 3699.json Created File: 3699.json
Created File: 4731.json Created File: 4731.json
Created File: 7460.json Created File: 7460.json
Created File: 861.json Created File: 861.json
Created File: 9176.json Created File: 9176.json
Created File: 6404.json Created File: 6404.json
Created File: 6207.json Created File: 6207.json
Created File: 1496.json Created File: 1496.json
Created File: 6335.json Created File: 6335.json
Created File: 3962.json Created File: 3962.json
Created File: 6296.json Created File: 6296.json
Created File: 8529.json Created File: 8529.json
Created File: 6311.json Created File: 6311.json
Created File: 8963.json Created File: 8963.json
Created File: 6202.json Created File: 6202.json
Created File: 5578.json Created File: 5578.json
Created File: 4959.json Created File: 4959.json
Created File: 6025.json Created File: 6025.json
Created File: 9983.json Created File: 9983.json
Created File: 1944.json Created File: 1944.json
Created File: 4286.json Created File: 4286.json
Created File: 1194.json Created File: 1194.json
Created File: 6662.json Created File: 6662.json
Created File: 403.json Created File: 403.json
Created File: 8288.json Created File: 8288.json
Created File: 1454.json Created File: 1454.json
Created File: 4580.json Created File: 4580.json
Created File: 8882.json Created File: 8882.json
Created File: 9091.json Created File: 9091.json
Created File: 4183.json Created File: 4183.json
Created File: 1488.json Created File: 1488.json
Created File: 1882.json Created File: 1882.json
Created File: 4558.json Created File: 4558.json
Created File: 2443.json Created File: 2443.json
Created File: 7725.json Created File: 7725.json
Created File: 4336.json Created File: 4336.json
Created File: 7143.json Created File: 7143.json
Created File: 5262.json Created File: 5262.json
Created File: 8446.json Created File: 8446.json
Created File: 8032.json Created File: 8032.json
Created File: 5570.json Created File: 5570.json
Created File: 1204.json Created File: 1204.json
Created File: 1410.json Created File: 1410.json
Created File: 1582.json Created File: 1582.json
Created File: 2079.json Created File: 2079.json
Created File: 1393.json Created File: 1393.json
Created File: 5298.json Created File: 5298.json
Created File: 10077.json Created File: 10077.json
Created File: 301.json Created File: 301.json
Created File: 5732.json Created File: 5732.json
Created File: 6082.json Created File: 6082.json
Created File: 534.json Created File: 534.json
Created File: 6141.json Created File: 6141.json
Created File: 8457.json Created File: 8457.json
Created File: 6806.json Created File: 6806.json
Created File: 9288.json Created File: 9288.json
Created File: 122.json Created File: 122.json
Created File: 9168.json Created File: 9168.json
Created File: 2550.json Created File: 2550.json
Created File: 9362.json Created File: 9362.json
Created File: 1395.json Created File: 1395.json
Created File: 4392.json Created File: 4392.json
Created File: 8723.json Created File: 8723.json
Created File: 4721.json Created File: 4721.json
Created File: 5729.json Created File: 5729.json
Created File: 10086.json Created File: 10086.json
Created File: 2111.json Created File: 2111.json
Created File: 8799.json Created File: 8799.json
Created File: 1032.json Created File: 1032.json
Created File: 8834.json Created File: 8834.json
Created File: 10127.json Created File: 10127.json
Created File: 3641.json Created File: 3641.json
Created File: 4334.json Created File: 4334.json
Created File: 5747.json Created File: 5747.json
Created File: 8412.json Created File: 8412.json
Created File: 8798.json Created File: 8798.json
Created File: 6741.json Created File: 6741.json
Created File: 1793.json Created File: 1793.json
Created File: 6308.json Created File: 6308.json
Created File: 1113.json Created File: 1113.json
Created File: 9047.json Created File: 9047.json
Created File: 1591.json Created File: 1591.json
Created File: 6417.json Created File: 6417.json
Created File: 5113.json Created File: 5113.json
Created File: 4914.json Created File: 4914.json
Created File: 1471.json Created File: 1471.json
Created File: 8448.json Created File: 8448.json
Created File: 5210.json Created File: 5210.json
Created File: 7158.json Created File: 7158.json
Created File: 6474.json Created File: 6474.json
Created File: 2411.json Created File: 2411.json
Created File: 9742.json Created File: 9742.json
Created File: 6387.json Created File: 6387.json
Created File: 353.json Created File: 353.json
Created File: 2318.json Created File: 2318.json
Created File: 6679.json Created File: 6679.json
Created File: 2156.json Created File: 2156.json
Created File: 5199.json Created File: 5199.json
Created File: 9711.json Created File: 9711.json
Created File: 8715.json Created File: 8715.json
Created File: 10102.json Created File: 10102.json
Created File: 5472.json Created File: 5472.json
Created File: 6177.json Created File: 6177.json
Created File: 6452.json Created File: 6452.json
Created File: 6921.json Created File: 6921.json
Created File: 6454.json Created File: 6454.json
Created File: 1590.json Created File: 1590.json
Created File: 9533.json Created File: 9533.json
Created File: 8496.json Created File: 8496.json
Created File: 1725.json Created File: 1725.json
Created File: 6547.json Created File: 6547.json
Created File: 9929.json Created File: 9929.json
Created File: 5329.json Created File: 5329.json
Created File: 10028.json Created File: 10028.json
Created File: 8286.json Created File: 8286.json
Created File: 1587.json Created File: 1587.json
Created File: 171.json Created File: 171.json
Created File: 3623.json Created File: 3623.json
Created File: 1303.json Created File: 1303.json
Created File: 2153.json Created File: 2153.json
Created File: 5494.json Created File: 5494.json
Created File: 3857.json Created File: 3857.json
Created File: 6645.json Created File: 6645.json
Created File: 8876.json Created File: 8876.json
Created File: 1908.json Created File: 1908.json
Created File: 3305.json Created File: 3305.json
Created File: 4151.json Created File: 4151.json
Created File: 4521.json Created File: 4521.json
Created File: 404.json Created File: 404.json
Created File: 4609.json Created File: 4609.json
Created File: 349.json Created File: 349.json
Created File: 8931.json Created File: 8931.json
Created File: 8659.json Created File: 8659.json
Created File: 5416.json Created File: 5416.json
Created File: 1631.json Created File: 1631.json
Created File: 5780.json Created File: 5780.json
Created File: 9041.json Created File: 9041.json
Created File: 4549.json Created File: 4549.json
Created File: 1279.json Created File: 1279.json
Created File: 2194.json Created File: 2194.json
Created File: 9137.json Created File: 9137.json
Created File: 3188.json Created File: 3188.json
Created File: 6351.json Created File: 6351.json
Created File: 9024.json Created File: 9024.json
Created File: 2087.json Created File: 2087.json
Created File: 6714.json Created File: 6714.json
Created File: 4724.json Created File: 4724.json
Created File: 7265.json Created File: 7265.json
Created File: 10000.json Created File: 10000.json
Created File: 3043.json Created File: 3043.json
Created File: 1962.json Created File: 1962.json
Created File: 9812.json Created File: 9812.json
Created File: 9028.json Created File: 9028.json
Created File: 4610.json Created File: 4610.json
Created File: 2117.json Created File: 2117.json
Created File: 9950.json Created File: 9950.json
Created File: 8470.json Created File: 8470.json
Created File: 8511.json Created File: 8511.json
Created File: 9923.json Created File: 9923.json
Created File: 5986.json Created File: 5986.json
Created File: 7145.json Created File: 7145.json
Created File: 3504.json Created File: 3504.json
Created File: 5139.json Created File: 5139.json
Created File: 8045.json Created File: 8045.json
Created File: 4423.json Created File: 4423.json
Created File: 4323.json Created File: 4323.json
Created File: 986.json Created File: 986.json
Created File: 4706.json Created File: 4706.json
Created File: 5450.json Created File: 5450.json
Created File: 1258.json Created File: 1258.json
Created File: 2182.json Created File: 2182.json
Created File: 951.json Created File: 951.json
Created File: 6371.json Created File: 6371.json
Created File: 3167.json Created File: 3167.json
Created File: 4988.json Created File: 4988.json
Created File: 6424.json Created File: 6424.json
Created File: 8856.json Created File: 8856.json
Created File: 4276.json Created File: 4276.json
Created File: 5375.json Created File: 5375.json
Created File: 8480.json Created File: 8480.json
Created File: 4975.json Created File: 4975.json
Created File: 23.json Created File: 23.json
Created File: 4978.json Created File: 4978.json
Created File: 7796.json Created File: 7796.json
Created File: 1289.json Created File: 1289.json
Created File: 4077.json Created File: 4077.json
Created File: 5748.json Created File: 5748.json
Created File: 4225.json Created File: 4225.json
Created File: 9198.json Created File: 9198.json
Created File: 6857.json Created File: 6857.json
Created File: 9687.json Created File: 9687.json
Created File: 1245.json Created File: 1245.json
Created File: 9086.json Created File: 9086.json
Created File: 4465.json Created File: 4465.json
Created File: 9206.json Created File: 9206.json
Created File: 7209.json Created File: 7209.json
Created File: 9276.json Created File: 9276.json
Created File: 8746.json Created File: 8746.json
Created File: 8597.json Created File: 8597.json
Created File: 7307.json Created File: 7307.json
Created File: 8028.json Created File: 8028.json
Created File: 7070.json Created File: 7070.json
Created File: 2288.json Created File: 2288.json
Created File: 4908.json Created File: 4908.json
Created File: 5451.json Created File: 5451.json
Created File: 4652.json Created File: 4652.json
Created File: 4017.json Created File: 4017.json
Created File: 9685.json Created File: 9685.json
Created File: 9087.json Created File: 9087.json
Created File: 6284.json Created File: 6284.json
Created File: 4518.json Created File: 4518.json
Created File: 7707.json Created File: 7707.json
Created File: 6789.json Created File: 6789.json
Created File: 7573.json Created File: 7573.json
Created File: 8742.json Created File: 8742.json
Created File: 4527.json Created File: 4527.json
Created File: 4693.json Created File: 4693.json
Created File: 7193.json Created File: 7193.json
Created File: 871.json Created File: 871.json
Created File: 3862.json Created File: 3862.json
Created File: 2996.json Created File: 2996.json
Created File: 6340.json Created File: 6340.json
Created File: 7541.json Created File: 7541.json
Created File: 4953.json Created File: 4953.json
Created File: 1420.json Created File: 1420.json
Created File: 5426.json Created File: 5426.json
Created File: 6359.json Created File: 6359.json
Created File: 5431.json Created File: 5431.json
Created File: 8960.json Created File: 8960.json
Created File: 4221.json Created File: 4221.json
Created File: 31.json Created File: 31.json
Created File: 5040.json Created File: 5040.json
Created File: 9570.json Created File: 9570.json
Created File: 5144.json Created File: 5144.json
Created File: 7275.json Created File: 7275.json
Created File: 1371.json Created File: 1371.json
Created File: 1035.json Created File: 1035.json
Created File: 9731.json Created File: 9731.json
Created File: 1840.json Created File: 1840.json
Created File: 3231.json Created File: 3231.json
Created File: 8518.json Created File: 8518.json
Created File: 6691.json Created File: 6691.json
Created File: 6012.json Created File: 6012.json
Created File: 8642.json Created File: 8642.json
Created File: 7641.json Created File: 7641.json
Created File: 1934.json Created File: 1934.json
Created File: 8887.json Created File: 8887.json
Created File: 1043.json Created File: 1043.json
Created File: 5161.json Created File: 5161.json
Created File: 4238.json Created File: 4238.json
Created File: 4136.json Created File: 4136.json
Created File: 5190.json Created File: 5190.json
Created File: 4562.json Created File: 4562.json
Created File: 9729.json Created File: 9729.json
Created File: 5488.json Created File: 5488.json
Created File: 3469.json Created File: 3469.json
Created File: 9456.json Created File: 9456.json
Created File: 8972.json Created File: 8972.json
Created File: 433.json Created File: 433.json
Created File: 6194.json Created File: 6194.json
Created File: 1535.json Created File: 1535.json
Created File: 1761.json Created File: 1761.json
Created File: 291.json Created File: 291.json
Created File: 1815.json Created File: 1815.json
Created File: 6458.json Created File: 6458.json
Created File: 3847.json Created File: 3847.json
Created File: 9962.json Created File: 9962.json
Created File: 1886.json Created File: 1886.json
Created File: 10021.json Created File: 10021.json
Created File: 10163.json Created File: 10163.json
Created File: 6241.json Created File: 6241.json
Created File: 5018.json Created File: 5018.json
Created File: 7124.json Created File: 7124.json
Created File: 5153.json Created File: 5153.json
Created File: 1648.json Created File: 1648.json
Created File: 5529.json Created File: 5529.json
Created File: 1089.json Created File: 1089.json
Created File: 2122.json Created File: 2122.json
Created File: 2924.json Created File: 2924.json
Created File: 6288.json Created File: 6288.json
Created File: 6286.json Created File: 6286.json
Created File: 6159.json Created File: 6159.json
Created File: 2987.json Created File: 2987.json
Created File: 9039.json Created File: 9039.json
Created File: 7138.json Created File: 7138.json
Created File: 5960.json Created File: 5960.json
Created File: 1639.json Created File: 1639.json
Created File: 8971.json Created File: 8971.json
Created File: 5313.json Created File: 5313.json
Created File: 2006.json Created File: 2006.json
Created File: 5489.json Created File: 5489.json
Created File: 7246.json Created File: 7246.json
Created File: 7203.json Created File: 7203.json
Created File: 9192.json Created File: 9192.json
Created File: 8460.json Created File: 8460.json
Created File: 9073.json Created File: 9073.json
Created File: 4306.json Created File: 4306.json
Created File: 8733.json Created File: 8733.json
Created File: 5582.json Created File: 5582.json
Created File: 7407.json Created File: 7407.json
Created File: 3294.json Created File: 3294.json
Created File: 2579.json Created File: 2579.json
Created File: 6058.json Created File: 6058.json
Created File: 5061.json Created File: 5061.json
Created File: 7552.json Created File: 7552.json
Created File: 4821.json Created File: 4821.json
Created File: 3400.json Created File: 3400.json
Created File: 4573.json Created File: 4573.json
Created File: 7881.json Created File: 7881.json
Created File: 1998.json Created File: 1998.json
Created File: 9256.json Created File: 9256.json
Created File: 3207.json Created File: 3207.json
Created File: 3895.json Created File: 3895.json
Created File: 10093.json Created File: 10093.json
Created File: 8427.json Created File: 8427.json
Created File: 1164.json Created File: 1164.json
Created File: 3960.json Created File: 3960.json
Created File: 1228.json Created File: 1228.json
Created File: 4717.json Created File: 4717.json
Created File: 8679.json Created File: 8679.json
Created File: 7429.json Created File: 7429.json
Created File: 3438.json Created File: 3438.json
Created File: 5041.json Created File: 5041.json
Created File: 5140.json Created File: 5140.json
Created File: 2238.json Created File: 2238.json
Created File: 1628.json Created File: 1628.json
Created File: 6876.json Created File: 6876.json
Created File: 8562.json Created File: 8562.json
Created File: 2134.json Created File: 2134.json
Created File: 5239.json Created File: 5239.json
Created File: 8663.json Created File: 8663.json
Created File: 85.json Created File: 85.json
Created File: 8726.json Created File: 8726.json
Created File: 6280.json Created File: 6280.json
Created File: 5176.json Created File: 5176.json
Created File: 6704.json Created File: 6704.json
Created File: 2451.json Created File: 2451.json
Created File: 8853.json Created File: 8853.json
Created File: 1455.json Created File: 1455.json
Created File: 5224.json Created File: 5224.json
Created File: 7180.json Created File: 7180.json
Created File: 4998.json Created File: 4998.json
Created File: 1519.json Created File: 1519.json
Created File: 9090.json Created File: 9090.json
Created File: 8424.json Created File: 8424.json
Created File: 6746.json Created File: 6746.json
Created File: 6172.json Created File: 6172.json
Created File: 5221.json Created File: 5221.json
Created File: 2973.json Created File: 2973.json
Created File: 4080.json Created File: 4080.json
Created File: 6947.json Created File: 6947.json
Created File: 7348.json Created File: 7348.json
Created File: 2168.json Created File: 2168.json
Created File: 2910.json Created File: 2910.json
Created File: 1634.json Created File: 1634.json
Created File: 2588.json Created File: 2588.json
Created File: 5191.json Created File: 5191.json
Created File: 5226.json Created File: 5226.json
Created File: 6005.json Created File: 6005.json
Created File: 997.json Created File: 997.json
Created File: 9646.json Created File: 9646.json
Created File: 9979.json Created File: 9979.json
Created File: 1318.json Created File: 1318.json
Created File: 4263.json Created File: 4263.json
Created File: 8987.json Created File: 8987.json
Created File: 5187.json Created File: 5187.json
Created File: 235.json Created File: 235.json
Created File: 4560.json Created File: 4560.json
Created File: 395.json Created File: 395.json
Created File: 6420.json Created File: 6420.json
Created File: 3149.json Created File: 3149.json
Created File: 5182.json Created File: 5182.json
Created File: 1560.json Created File: 1560.json
Created File: 5784.json Created File: 5784.json
Created File: 9769.json Created File: 9769.json
Created File: 5342.json Created File: 5342.json
Created File: 4666.json Created File: 4666.json
Created File: 8380.json Created File: 8380.json
Created File: 1977.json Created File: 1977.json
Created File: 4143.json Created File: 4143.json
Created File: 10050.json Created File: 10050.json
Created File: 6322.json Created File: 6322.json
Created File: 142.json Created File: 142.json
Created File: 9098.json Created File: 9098.json
Created File: 5469.json Created File: 5469.json
Created File: 6369.json Created File: 6369.json
Created File: 3181.json Created File: 3181.json
Created File: 6803.json Created File: 6803.json
Created File: 6188.json Created File: 6188.json
Created File: 6914.json Created File: 6914.json
Created File: 6718.json Created File: 6718.json
Created File: 3035.json Created File: 3035.json
Created File: 4593.json Created File: 4593.json
Created File: 5485.json Created File: 5485.json
Created File: 545.json Created File: 545.json
Created File: 6999.json Created File: 6999.json
Created File: 9194.json Created File: 9194.json
Created File: 8656.json Created File: 8656.json
Created File: 4824.json Created File: 4824.json
Created File: 5883.json Created File: 5883.json
Created File: 1210.json Created File: 1210.json
Created File: 4849.json Created File: 4849.json
Created File: 8393.json Created File: 8393.json
Created File: 82.json Created File: 82.json
Created File: 4891.json Created File: 4891.json
Created File: 5242.json Created File: 5242.json
Created File: 1884.json Created File: 1884.json
Created File: 3332.json Created File: 3332.json
Created File: 1027.json Created File: 1027.json
Created File: 7119.json Created File: 7119.json
Created File: 8818.json Created File: 8818.json
Created File: 7294.json Created File: 7294.json
Created File: 7120.json Created File: 7120.json
Created File: 1287.json Created File: 1287.json
Created File: 9023.json Created File: 9023.json
Created File: 8989.json Created File: 8989.json
Created File: 10014.json Created File: 10014.json
Created File: 4496.json Created File: 4496.json
Created File: 1508.json Created File: 1508.json
Created File: 975.json Created File: 975.json
Created File: 3600.json Created File: 3600.json
Created File: 3727.json Created File: 3727.json
Created File: 7651.json Created File: 7651.json
Created File: 9187.json Created File: 9187.json
Created File: 8904.json Created File: 8904.json
Created File: 4986.json Created File: 4986.json
Created File: 1531.json Created File: 1531.json
Created File: 1578.json Created File: 1578.json
Created File: 220.json Created File: 220.json
Created File: 6290.json Created File: 6290.json
Created File: 1359.json Created File: 1359.json
Created File: 2349.json Created File: 2349.json
Created File: 4794.json Created File: 4794.json
Created File: 6136.json Created File: 6136.json
Created File: 665.json Created File: 665.json
Created File: 1489.json Created File: 1489.json
Created File: 5543.json Created File: 5543.json
Created File: 3168.json Created File: 3168.json
Created File: 8785.json Created File: 8785.json
Created File: 5145.json Created File: 5145.json
Created File: 6443.json Created File: 6443.json
Created File: 4832.json Created File: 4832.json
Created File: 6236.json Created File: 6236.json
Created File: 4996.json Created File: 4996.json
Created File: 1297.json Created File: 1297.json
Created File: 5750.json Created File: 5750.json
Created File: 5640.json Created File: 5640.json
Created File: 5146.json Created File: 5146.json
Created File: 10033.json Created File: 10033.json
Created File: 1749.json Created File: 1749.json
Created File: 10096.json Created File: 10096.json
Created File: 2882.json Created File: 2882.json
Created File: 5247.json Created File: 5247.json
Created File: 1092.json Created File: 1092.json
Created File: 2870.json Created File: 2870.json
Created File: 4135.json Created File: 4135.json
Created File: 1246.json Created File: 1246.json
Created File: 4916.json Created File: 4916.json
Created File: 4374.json Created File: 4374.json
Created File: 5001.json Created File: 5001.json
Created File: 5896.json Created File: 5896.json
Created File: 3184.json Created File: 3184.json
Created File: 4146.json Created File: 4146.json
Created File: 7510.json Created File: 7510.json
Created File: 4554.json Created File: 4554.json
Created File: 4920.json Created File: 4920.json
Created File: 6958.json Created File: 6958.json
Created File: 7238.json Created File: 7238.json
Created File: 394.json Created File: 394.json
Created File: 1914.json Created File: 1914.json
Created File: 2065.json Created File: 2065.json
Created File: 6574.json Created File: 6574.json
Created File: 6952.json Created File: 6952.json
Created File: 1361.json Created File: 1361.json
Created File: 8692.json Created File: 8692.json
Created File: 9119.json Created File: 9119.json
Created File: 1422.json Created File: 1422.json
Created File: 4150.json Created File: 4150.json
Created File: 10055.json Created File: 10055.json
Created File: 2296.json Created File: 2296.json
Created File: 6642.json Created File: 6642.json
Created File: 2529.json Created File: 2529.json
Created File: 3854.json Created File: 3854.json
Created File: 4798.json Created File: 4798.json
Created File: 1470.json Created File: 1470.json
Created File: 9739.json Created File: 9739.json
Created File: 4054.json Created File: 4054.json
Created File: 8376.json Created File: 8376.json
Created File: 5240.json Created File: 5240.json
Created File: 4861.json Created File: 4861.json
Created File: 8968.json Created File: 8968.json
Created File: 4346.json Created File: 4346.json
Created File: 4782.json Created File: 4782.json
Created File: 2174.json Created File: 2174.json
Created File: 8712.json Created File: 8712.json
Created File: 66.json Created File: 66.json
Created File: 4960.json Created File: 4960.json
Created File: 3524.json Created File: 3524.json
Created File: 2073.json Created File: 2073.json
Created File: 9184.json Created File: 9184.json
Created File: 2570.json Created File: 2570.json
Created File: 7423.json Created File: 7423.json
Created File: 1384.json Created File: 1384.json
Created File: 6180.json Created File: 6180.json
Created File: 8499.json Created File: 8499.json
Created File: 365.json Created File: 365.json
Created File: 516.json Created File: 516.json
Created File: 6576.json Created File: 6576.json
Created File: 5510.json Created File: 5510.json
Created File: 4197.json Created File: 4197.json
Created File: 8572.json Created File: 8572.json
Created File: 6110.json Created File: 6110.json
Created File: 5243.json Created File: 5243.json
Created File: 4720.json Created File: 4720.json
Created File: 3846.json Created File: 3846.json
Created File: 7187.json Created File: 7187.json
Created File: 6844.json Created File: 6844.json
Created File: 2613.json Created File: 2613.json
Created File: 3090.json Created File: 3090.json
Created File: 6494.json Created File: 6494.json
Created File: 5459.json Created File: 5459.json
Created File: 9302.json Created File: 9302.json
Created File: 6088.json Created File: 6088.json
Created File: 3521.json Created File: 3521.json
Created File: 6624.json Created File: 6624.json
Created File: 7174.json Created File: 7174.json
Created File: 56.json Created File: 56.json
Created File: 4643.json Created File: 4643.json
Created File: 2061.json Created File: 2061.json
Created File: 4699.json Created File: 4699.json
Created File: 10157.json Created File: 10157.json
Created File: 1281.json Created File: 1281.json
Created File: 5901.json Created File: 5901.json
Created File: 9210.json Created File: 9210.json
Created File: 1377.json Created File: 1377.json
Created File: 6195.json Created File: 6195.json
Created File: 4722.json Created File: 4722.json
Created File: 6163.json Created File: 6163.json
Created File: 9848.json Created File: 9848.json
Created File: 371.json Created File: 371.json
Created File: 6090.json Created File: 6090.json
Created File: 8878.json Created File: 8878.json
Created File: 5354.json Created File: 5354.json
Created File: 9841.json Created File: 9841.json
Created File: 5539.json Created File: 5539.json
Created File: 1991.json Created File: 1991.json
Created File: 10008.json Created File: 10008.json
Created File: 4309.json Created File: 4309.json
Created File: 4261.json Created File: 4261.json
Created File: 3272.json Created File: 3272.json
Created File: 10173.json Created File: 10173.json
Created File: 2295.json Created File: 2295.json
Created File: 5204.json Created File: 5204.json
Created File: 9065.json Created File: 9065.json
Created File: 6112.json Created File: 6112.json
Created File: 5165.json Created File: 5165.json
Created File: 2867.json Created File: 2867.json
Created File: 1646.json Created File: 1646.json
Created File: 5024.json Created File: 5024.json
Created File: 4314.json Created File: 4314.json
Created File: 3706.json Created File: 3706.json
Created File: 1811.json Created File: 1811.json
Created File: 1187.json Created File: 1187.json
Created File: 468.json Created File: 468.json
Created File: 8504.json Created File: 8504.json
Created File: 3929.json Created File: 3929.json
Created File: 4174.json Created File: 4174.json
Created File: 7086.json Created File: 7086.json
Created File: 6562.json Created File: 6562.json
Created File: 8779.json Created File: 8779.json
Created File: 8979.json Created File: 8979.json
Created File: 2898.json Created File: 2898.json
Created File: 8023.json Created File: 8023.json
Created File: 9275.json Created File: 9275.json
Created File: 9054.json Created File: 9054.json
Created File: 3465.json Created File: 3465.json
Created File: 4493.json Created File: 4493.json
Created File: 6080.json Created File: 6080.json
Created File: 1538.json Created File: 1538.json
Created File: 5905.json Created File: 5905.json
Created File: 4697.json Created File: 4697.json
Created File: 8965.json Created File: 8965.json
Created File: 10160.json Created File: 10160.json
Created File: 4943.json Created File: 4943.json
Created File: 2464.json Created File: 2464.json
Created File: 8278.json Created File: 8278.json
Created File: 10143.json Created File: 10143.json
Created File: 5537.json Created File: 5537.json
Created File: 7081.json Created File: 7081.json
Created File: 2537.json Created File: 2537.json
Created File: 1812.json Created File: 1812.json
Created File: 4112.json Created File: 4112.json
Created File: 9945.json Created File: 9945.json
Created File: 9079.json Created File: 9079.json
Created File: 3135.json Created File: 3135.json
Created File: 1895.json Created File: 1895.json
Created File: 567.json Created File: 567.json
Created File: 8739.json Created File: 8739.json
Created File: 4650.json Created File: 4650.json
Created File: 2467.json Created File: 2467.json
Created File: 5264.json Created File: 5264.json
Created File: 1951.json Created File: 1951.json
Created File: 8546.json Created File: 8546.json
Created File: 6230.json Created File: 6230.json
Created File: 1443.json Created File: 1443.json
Created File: 1559.json Created File: 1559.json
Created File: 4265.json Created File: 4265.json
Created File: 6282.json Created File: 6282.json
Created File: 2353.json Created File: 2353.json
Created File: 4311.json Created File: 4311.json
Created File: 494.json Created File: 494.json
Created File: 7901.json Created File: 7901.json
Created File: 6225.json Created File: 6225.json
Created File: 4051.json Created File: 4051.json
Created File: 1440.json Created File: 1440.json
Created File: 5566.json Created File: 5566.json
Created File: 3678.json Created File: 3678.json
Created File: 10036.json Created File: 10036.json
Created File: 192.json Created File: 192.json
Created File: 4448.json Created File: 4448.json
Created File: 7071.json Created File: 7071.json
Created File: 5020.json Created File: 5020.json
Created File: 1306.json Created File: 1306.json
Created File: 1336.json Created File: 1336.json
Created File: 3811.json Created File: 3811.json
Created File: 3017.json Created File: 3017.json
Created File: 1467.json Created File: 1467.json
Created File: 1261.json Created File: 1261.json
Created File: 4612.json Created File: 4612.json
Created File: 25.json Created File: 25.json
Created File: 9921.json Created File: 9921.json
Created File: 5704.json Created File: 5704.json
Created File: 4664.json Created File: 4664.json
Created File: 9144.json Created File: 9144.json
Created File: 7371.json Created File: 7371.json
Created File: 3166.json Created File: 3166.json
Created File: 7256.json Created File: 7256.json
Created File: 539.json Created File: 539.json
Created File: 4099.json Created File: 4099.json
Created File: 4450.json Created File: 4450.json
Created File: 5046.json Created File: 5046.json
Created File: 1633.json Created File: 1633.json
Created File: 3880.json Created File: 3880.json
Created File: 3701.json Created File: 3701.json
Created File: 95.json Created File: 95.json
Created File: 3788.json Created File: 3788.json
Created File: 4532.json Created File: 4532.json
Created File: 4164.json Created File: 4164.json
Created File: 2520.json Created File: 2520.json
Created File: 6928.json Created File: 6928.json
Created File: 6955.json Created File: 6955.json
Created File: 6339.json Created File: 6339.json
Created File: 4962.json Created File: 4962.json
Created File: 5922.json Created File: 5922.json
Created File: 6894.json Created File: 6894.json
Created File: 2303.json Created File: 2303.json
Created File: 6199.json Created File: 6199.json
Created File: 978.json Created File: 978.json
Created File: 10001.json Created File: 10001.json
Created File: 8648.json Created File: 8648.json
Created File: 5365.json Created File: 5365.json
Created File: 6751.json Created File: 6751.json
Created File: 1513.json Created File: 1513.json
Created File: 6447.json Created File: 6447.json
Created File: 1946.json Created File: 1946.json
Created File: 3562.json Created File: 3562.json
Created File: 5705.json Created File: 5705.json
Created File: 2231.json Created File: 2231.json
Created File: 10010.json Created File: 10010.json
Created File: 1929.json Created File: 1929.json
Created File: 5189.json Created File: 5189.json
Created File: 3625.json Created File: 3625.json
Created File: 4166.json Created File: 4166.json
Created File: 6792.json Created File: 6792.json
Created File: 4357.json Created File: 4357.json
Created File: 4999.json Created File: 4999.json
Created File: 6327.json Created File: 6327.json
Created File: 4355.json Created File: 4355.json
Created File: 2425.json Created File: 2425.json
Created File: 4213.json Created File: 4213.json
Created File: 4087.json Created File: 4087.json
Created File: 1675.json Created File: 1675.json
Created File: 4372.json Created File: 4372.json
Created File: 7786.json Created File: 7786.json
Created File: 6989.json Created File: 6989.json
Created File: 8682.json Created File: 8682.json
Created File: 6556.json Created File: 6556.json
Created File: 6155.json Created File: 6155.json
Created File: 5287.json Created File: 5287.json
Created File: 8939.json Created File: 8939.json
Created File: 1066.json Created File: 1066.json
Created File: 5428.json Created File: 5428.json
Created File: 8374.json Created File: 8374.json
Created File: 676.json Created File: 676.json
Created File: 8820.json Created File: 8820.json
Created File: 6438.json Created File: 6438.json
Created File: 1498.json Created File: 1498.json
Created File: 5509.json Created File: 5509.json
Created File: 9910.json Created File: 9910.json
Created File: 1088.json Created File: 1088.json
Created File: 2805.json Created File: 2805.json
Created File: 2847.json Created File: 2847.json
Created File: 5482.json Created File: 5482.json
Created File: 2219.json Created File: 2219.json
Created File: 8614.json Created File: 8614.json
Created File: 7506.json Created File: 7506.json
Created File: 5423.json Created File: 5423.json
Created File: 8372.json Created File: 8372.json
Created File: 3989.json Created File: 3989.json
Created File: 1568.json Created File: 1568.json
Created File: 8757.json Created File: 8757.json
Created File: 9789.json Created File: 9789.json
Created File: 343.json Created File: 343.json
Created File: 4137.json Created File: 4137.json
Created File: 7336.json Created File: 7336.json
Created File: 3826.json Created File: 3826.json
Created File: 5504.json Created File: 5504.json
Created File: 1807.json Created File: 1807.json
Created File: 209.json Created File: 209.json
Created File: 3835.json Created File: 3835.json
Created File: 1348.json Created File: 1348.json
Created File: 4992.json Created File: 4992.json
Created File: 7354.json Created File: 7354.json
Created File: 2325.json Created File: 2325.json
Created File: 3195.json Created File: 3195.json
Created File: 3111.json Created File: 3111.json
Created File: 5088.json Created File: 5088.json
Created File: 6087.json Created File: 6087.json
Created File: 3157.json Created File: 3157.json
Created File: 3280.json Created File: 3280.json
Created File: 1875.json Created File: 1875.json
Created File: 1170.json Created File: 1170.json
Created File: 9705.json Created File: 9705.json
Created File: 7195.json Created File: 7195.json
Created File: 5207.json Created File: 5207.json
Created File: 4209.json Created File: 4209.json
Created File: 2167.json Created File: 2167.json
Created File: 7133.json Created File: 7133.json
Created File: 8390.json Created File: 8390.json
Created File: 1291.json Created File: 1291.json
Created File: 4236.json Created File: 4236.json
Created File: 1062.json Created File: 1062.json
Created File: 6657.json Created File: 6657.json
Created File: 1337.json Created File: 1337.json
Created File: 10095.json Created File: 10095.json
Created File: 2762.json Created File: 2762.json
Created File: 3182.json Created File: 3182.json
Created File: 4686.json Created File: 4686.json
Created File: 8636.json Created File: 8636.json
Created File: 9245.json Created File: 9245.json
Created File: 7406.json Created File: 7406.json
Created File: 5966.json Created File: 5966.json
Created File: 3760.json Created File: 3760.json
Created File: 8632.json Created File: 8632.json
Created File: 3186.json Created File: 3186.json
Created File: 1135.json Created File: 1135.json
Created File: 4431.json Created File: 4431.json
Created File: 4547.json Created File: 4547.json
Created File: 6066.json Created File: 6066.json
Created File: 6461.json Created File: 6461.json
Created File: 5361.json Created File: 5361.json
Created File: 2279.json Created File: 2279.json
Created File: 6440.json Created File: 6440.json
Created File: 4303.json Created File: 4303.json
Created File: 5433.json Created File: 5433.json
Created File: 2270.json Created File: 2270.json
Created File: 7142.json Created File: 7142.json
Created File: 6886.json Created File: 6886.json
Created File: 9697.json Created File: 9697.json
Created File: 4258.json Created File: 4258.json
Created File: 8406.json Created File: 8406.json
Created File: 8918.json Created File: 8918.json
Created File: 9102.json Created File: 9102.json
Created File: 1565.json Created File: 1565.json
Created File: 5447.json Created File: 5447.json
Created File: 5122.json Created File: 5122.json
Created File: 3980.json Created File: 3980.json
Created File: 2779.json Created File: 2779.json
Created File: 10111.json Created File: 10111.json
Created File: 2610.json Created File: 2610.json
Created File: 1382.json Created File: 1382.json
Created File: 2574.json Created File: 2574.json
Created File: 1993.json Created File: 1993.json
Created File: 6299.json Created File: 6299.json
Created File: 6603.json Created File: 6603.json
Created File: 9135.json Created File: 9135.json
Created File: 4712.json Created File: 4712.json
Created File: 2844.json Created File: 2844.json
Created File: 4245.json Created File: 4245.json
Created File: 8930.json Created File: 8930.json
Created File: 5320.json Created File: 5320.json
Created File: 9886.json Created File: 9886.json
Created File: 2239.json Created File: 2239.json
Created File: 4048.json Created File: 4048.json
Created File: 8842.json Created File: 8842.json
Created File: 5449.json Created File: 5449.json
Created File: 1723.json Created File: 1723.json
Created File: 3500.json Created File: 3500.json
Created File: 4936.json Created File: 4936.json
Created File: 8664.json Created File: 8664.json
Created File: 10132.json Created File: 10132.json
Created File: 2813.json Created File: 2813.json
Created File: 1270.json Created File: 1270.json
Created File: 1472.json Created File: 1472.json
Created File: 5611.json Created File: 5611.json
Created File: 8761.json Created File: 8761.json
Created File: 8833.json Created File: 8833.json
Created File: 8151.json Created File: 8151.json
Created File: 4945.json Created File: 4945.json
Created File: 632.json Created File: 632.json
Created File: 1862.json Created File: 1862.json
Created File: 7361.json Created File: 7361.json
Created File: 234.json Created File: 234.json
Created File: 7066.json Created File: 7066.json
Created File: 4219.json Created File: 4219.json
Created File: 417.json Created File: 417.json
Created File: 719.json Created File: 719.json
Created File: 1607.json Created File: 1607.json
Created File: 2335.json Created File: 2335.json
Created File: 5902.json Created File: 5902.json
Created File: 10149.json Created File: 10149.json
Created File: 3119.json Created File: 3119.json
Created File: 3477.json Created File: 3477.json
Created File: 1517.json Created File: 1517.json
Created File: 9277.json Created File: 9277.json
Created File: 5840.json Created File: 5840.json
Created File: 2938.json Created File: 2938.json
Created File: 9151.json Created File: 9151.json
Created File: 3010.json Created File: 3010.json
Created File: 9138.json Created File: 9138.json
Created File: 4256.json Created File: 4256.json
Created File: 5911.json Created File: 5911.json
Created File: 2909.json Created File: 2909.json
Created File: 3994.json Created File: 3994.json
Created File: 6344.json Created File: 6344.json
Created File: 4648.json Created File: 4648.json
Created File: 1180.json Created File: 1180.json
Created File: 5410.json Created File: 5410.json
Created File: 1153.json Created File: 1153.json
Created File: 6532.json Created File: 6532.json
Created File: 10047.json Created File: 10047.json
Created File: 5815.json Created File: 5815.json
Created File: 3795.json Created File: 3795.json
Created File: 8392.json Created File: 8392.json
Created File: 2908.json Created File: 2908.json
Created File: 5532.json Created File: 5532.json
Created File: 1293.json Created File: 1293.json
Created File: 8896.json Created File: 8896.json
Created File: 2114.json Created File: 2114.json
Created File: 6439.json Created File: 6439.json
Created File: 8443.json Created File: 8443.json
Created File: 1324.json Created File: 1324.json
Created File: 8984.json Created File: 8984.json
Created File: 2086.json Created File: 2086.json
Created File: 7418.json Created File: 7418.json
Created File: 5444.json Created File: 5444.json
Created File: 5843.json Created File: 5843.json
Created File: 6727.json Created File: 6727.json
Created File: 10144.json Created File: 10144.json
Created File: 4154.json Created File: 4154.json
Created File: 6364.json Created File: 6364.json
Created File: 8748.json Created File: 8748.json
Created File: 6956.json Created File: 6956.json
Created File: 8428.json Created File: 8428.json
Created File: 4566.json Created File: 4566.json
Created File: 6922.json Created File: 6922.json
Created File: 10042.json Created File: 10042.json
Created File: 1301.json Created File: 1301.json
Created File: 5094.json Created File: 5094.json
Created File: 5982.json Created File: 5982.json
Created File: 9718.json Created File: 9718.json
Created File: 2046.json Created File: 2046.json
Created File: 8520.json Created File: 8520.json
Created File: 9297.json Created File: 9297.json
Created File: 4740.json Created File: 4740.json
Created File: 38.json Created File: 38.json
Created File: 6031.json Created File: 6031.json
Created File: 4836.json Created File: 4836.json
Created File: 3328.json Created File: 3328.json
Created File: 5770.json Created File: 5770.json
Created File: 3317.json Created File: 3317.json
Created File: 5100.json Created File: 5100.json
Created File: 6858.json Created File: 6858.json
Created File: 10108.json Created File: 10108.json
Created File: 7384.json Created File: 7384.json
Created File: 9680.json Created File: 9680.json
Created File: 17.json Created File: 17.json
Created File: 9917.json Created File: 9917.json
Created File: 5025.json Created File: 5025.json
Created File: 3315.json Created File: 3315.json
Created File: 295.json Created File: 295.json
Created File: 3901.json Created File: 3901.json
Created File: 1722.json Created File: 1722.json
Created File: 9211.json Created File: 9211.json
Created File: 160.json Created File: 160.json
Created File: 7114.json Created File: 7114.json
Created File: 8646.json Created File: 8646.json
Created File: 9733.json Created File: 9733.json
Created File: 4934.json Created File: 4934.json
Created File: 5159.json Created File: 5159.json
Created File: 1330.json Created File: 1330.json
Created File: 3478.json Created File: 3478.json
Created File: 7243.json Created File: 7243.json
Created File: 9755.json Created File: 9755.json
Created File: 4251.json Created File: 4251.json
Created File: 650.json Created File: 650.json
Created File: 2307.json Created File: 2307.json
Created File: 10087.json Created File: 10087.json
Created File: 6256.json Created File: 6256.json
Created File: 2301.json Created File: 2301.json
Created File: 5545.json Created File: 5545.json
Created File: 6658.json Created File: 6658.json
Created File: 5827.json Created File: 5827.json
Created File: 4326.json Created File: 4326.json
Created File: 3886.json Created File: 3886.json
Created File: 7443.json Created File: 7443.json
Created File: 6283.json Created File: 6283.json
Created File: 4478.json Created File: 4478.json
Created File: 9761.json Created File: 9761.json
Created File: 1810.json Created File: 1810.json
Created File: 1362.json Created File: 1362.json
Created File: 2984.json Created File: 2984.json
Created File: 7002.json Created File: 7002.json
Created File: 6119.json Created File: 6119.json
Created File: 5805.json Created File: 5805.json
Created File: 2348.json Created File: 2348.json
Created File: 2419.json Created File: 2419.json
Created File: 303.json Created File: 303.json
Created File: 6198.json Created File: 6198.json
Created File: 6305.json Created File: 6305.json
Created File: 6421.json Created File: 6421.json
Created File: 7075.json Created File: 7075.json
Created File: 3212.json Created File: 3212.json
Created File: 8974.json Created File: 8974.json
Created File: 2331.json Created File: 2331.json
Created File: 6650.json Created File: 6650.json
Created File: 7866.json Created File: 7866.json
Created File: 1996.json Created File: 1996.json
Created File: 2484.json Created File: 2484.json
Created File: 5656.json Created File: 5656.json
Created File: 5125.json Created File: 5125.json
Created File: 6407.json Created File: 6407.json
Created File: 5941.json Created File: 5941.json
Created File: 8506.json Created File: 8506.json
Created File: 6353.json Created File: 6353.json
Created File: 6042.json Created File: 6042.json
Created File: 6403.json Created File: 6403.json
Created File: 7467.json Created File: 7467.json
Created File: 8515.json Created File: 8515.json
Created File: 5291.json Created File: 5291.json
Created File: 7349.json Created File: 7349.json
Created File: 9026.json Created File: 9026.json
Created File: 1698.json Created File: 1698.json
Created File: 2921.json Created File: 2921.json
Created File: 6337.json Created File: 6337.json
Created File: 4834.json Created File: 4834.json
Created File: 1852.json Created File: 1852.json
Created File: 6548.json Created File: 6548.json
Created File: 2158.json Created File: 2158.json
Created File: 5346.json Created File: 5346.json
Created File: 2955.json Created File: 2955.json
Created File: 2761.json Created File: 2761.json
Created File: 2572.json Created File: 2572.json
Created File: 1817.json Created File: 1817.json
Created File: 6142.json Created File: 6142.json
Created File: 3708.json Created File: 3708.json
Created File: 5135.json Created File: 5135.json
Created File: 873.json Created File: 873.json
Created File: 4981.json Created File: 4981.json
Created File: 2393.json Created File: 2393.json
Created File: 4414.json Created File: 4414.json
Created File: 3755.json Created File: 3755.json
Created File: 5200.json Created File: 5200.json
Created File: 6846.json Created File: 6846.json
Created File: 6002.json Created File: 6002.json
Created File: 4669.json Created File: 4669.json
Created File: 7422.json Created File: 7422.json
Created File: 8728.json Created File: 8728.json
Created File: 5281.json Created File: 5281.json
Created File: 8570.json Created File: 8570.json
Created File: 5272.json Created File: 5272.json
Created File: 5311.json Created File: 5311.json
Created File: 6292.json Created File: 6292.json
Created File: 9689.json Created File: 9689.json
Created File: 4474.json Created File: 4474.json
Created File: 3330.json Created File: 3330.json
Created File: 6472.json Created File: 6472.json
Created File: 1936.json Created File: 1936.json
Created File: 6394.json Created File: 6394.json
Created File: 6162.json Created File: 6162.json
Created File: 6231.json Created File: 6231.json
Created File: 8957.json Created File: 8957.json
Created File: 7240.json Created File: 7240.json
Created File: 8777.json Created File: 8777.json
Created File: 1490.json Created File: 1490.json
Created File: 3858.json Created File: 3858.json
Created File: 9714.json Created File: 9714.json
Created File: 1715.json Created File: 1715.json
Created File: 3413.json Created File: 3413.json
Created File: 1873.json Created File: 1873.json
Created File: 8940.json Created File: 8940.json
Created File: 7094.json Created File: 7094.json
Created File: 6676.json Created File: 6676.json
Created File: 4139.json Created File: 4139.json
Created File: 6719.json Created File: 6719.json
Created File: 5585.json Created File: 5585.json
Created File: 4692.json Created File: 4692.json
Created File: 6400.json Created File: 6400.json
Created File: 7004.json Created File: 7004.json
Created File: 7127.json Created File: 7127.json
Created File: 2002.json Created File: 2002.json
Created File: 7921.json Created File: 7921.json
Created File: 2242.json Created File: 2242.json
Created File: 4941.json Created File: 4941.json
Created File: 6726.json Created File: 6726.json
Created File: 1481.json Created File: 1481.json
Created File: 1201.json Created File: 1201.json
Created File: 4352.json Created File: 4352.json
Created File: 6973.json Created File: 6973.json
Created File: 2180.json Created File: 2180.json
Created File: 8776.json Created File: 8776.json
Created File: 8468.json Created File: 8468.json
Created File: 2643.json Created File: 2643.json
Created File: 8564.json Created File: 8564.json
Created File: 8549.json Created File: 8549.json
Created File: 9926.json Created File: 9926.json
Created File: 8710.json Created File: 8710.json
Created File: 4373.json Created File: 4373.json
Created File: 8845.json Created File: 8845.json
Created File: 8880.json Created File: 8880.json
Created File: 1008.json Created File: 1008.json
Created File: 4910.json Created File: 4910.json
Created File: 3664.json Created File: 3664.json
Created File: 742.json Created File: 742.json
Created File: 6524.json Created File: 6524.json
Created File: 2316.json Created File: 2316.json
Created File: 649.json Created File: 649.json
Created File: 5552.json Created File: 5552.json
Created File: 1351.json Created File: 1351.json
Created File: 9678.json Created File: 9678.json
Created File: 9730.json Created File: 9730.json
Created File: 6824.json Created File: 6824.json
Created File: 10071.json Created File: 10071.json
Created File: 6009.json Created File: 6009.json
Created File: 3984.json Created File: 3984.json
Created File: 3859.json Created File: 3859.json
Created File: 1585.json Created File: 1585.json
Created File: 5369.json Created File: 5369.json
Created File: 7302.json Created File: 7302.json
Created File: 3542.json Created File: 3542.json
Created File: 2017.json Created File: 2017.json
Created File: 1550.json Created File: 1550.json
Created File: 7910.json Created File: 7910.json
Created File: 6079.json Created File: 6079.json
Created File: 5474.json Created File: 5474.json
Created File: 4795.json Created File: 4795.json
Created File: 8932.json Created File: 8932.json
Created File: 5084.json Created File: 5084.json
Created File: 8967.json Created File: 8967.json
Created File: 8430.json Created File: 8430.json
Created File: 8513.json Created File: 8513.json
Created File: 9013.json Created File: 9013.json
Created File: 4620.json Created File: 4620.json
Created File: 2580.json Created File: 2580.json
Created File: 5180.json Created File: 5180.json
Created File: 4598.json Created File: 4598.json
Created File: 1967.json Created File: 1967.json
Created File: 3356.json Created File: 3356.json
Created File: 7716.json Created File: 7716.json
Created File: 7331.json Created File: 7331.json
Created File: 9027.json Created File: 9027.json
Created File: 1844.json Created File: 1844.json
Created File: 9315.json Created File: 9315.json
Created File: 4887.json Created File: 4887.json
Created File: 2398.json Created File: 2398.json
Created File: 5253.json Created File: 5253.json
Created File: 5407.json Created File: 5407.json
Created File: 10130.json Created File: 10130.json
Created File: 10060.json Created File: 10060.json
Created File: 3957.json Created File: 3957.json
Created File: 2067.json Created File: 2067.json
Created File: 8985.json Created File: 8985.json
Created File: 6298.json Created File: 6298.json
Created File: 8551.json Created File: 8551.json
Created File: 5325.json Created File: 5325.json
Created File: 7244.json Created File: 7244.json
Created File: 9830.json Created File: 9830.json
Created File: 4702.json Created File: 4702.json
Created File: 7151.json Created File: 7151.json
Created File: 4931.json Created File: 4931.json
Created File: 1611.json Created File: 1611.json
Created File: 1474.json Created File: 1474.json
Created File: 9712.json Created File: 9712.json
Created File: 7067.json Created File: 7067.json
Created File: 9542.json Created File: 9542.json
Created File: 993.json Created File: 993.json
Created File: 5519.json Created File: 5519.json
Created File: 1805.json Created File: 1805.json
Created File: 5283.json Created File: 5283.json
Created File: 2534.json Created File: 2534.json
Created File: 273.json Created File: 273.json
Created File: 2337.json Created File: 2337.json
Created File: 2793.json Created File: 2793.json
Created File: 7826.json Created File: 7826.json
Created File: 4696.json Created File: 4696.json
Created File: 4600.json Created File: 4600.json
Created File: 7396.json Created File: 7396.json
Created File: 1657.json Created File: 1657.json
Created File: 7850.json Created File: 7850.json
Created File: 6164.json Created File: 6164.json
Created File: 5774.json Created File: 5774.json
Created File: 9700.json Created File: 9700.json
Created File: 6814.json Created File: 6814.json
Created File: 9150.json Created File: 9150.json
Created File: 9057.json Created File: 9057.json
Created File: 5029.json Created File: 5029.json
Created File: 5225.json Created File: 5225.json
Created File: 4976.json Created File: 4976.json
Created File: 5303.json Created File: 5303.json
Created File: 6631.json Created File: 6631.json
Created File: 4637.json Created File: 4637.json
Created File: 3052.json Created File: 3052.json
Created File: 1149.json Created File: 1149.json
Created File: 2471.json Created File: 2471.json
Created File: 6214.json Created File: 6214.json
Created File: 7072.json Created File: 7072.json
Created File: 4407.json Created File: 4407.json
Created File: 4670.json Created File: 4670.json
Created File: 1874.json Created File: 1874.json
Created File: 8789.json Created File: 8789.json
Created File: 3214.json Created File: 3214.json
Created File: 3485.json Created File: 3485.json
Created File: 14.json Created File: 14.json
Created File: 3056.json Created File: 3056.json
Created File: 6151.json Created File: 6151.json
Created File: 3762.json Created File: 3762.json
Created File: 6161.json Created File: 6161.json
Created File: 1103.json Created File: 1103.json
Created File: 7824.json Created File: 7824.json
Created File: 8657.json Created File: 8657.json
Created File: 8618.json Created File: 8618.json
Created File: 5318.json Created File: 5318.json
Created File: 6183.json Created File: 6183.json
Created File: 6368.json Created File: 6368.json
Created File: 4321.json Created File: 4321.json
Created File: 1877.json Created File: 1877.json
Created File: 2253.json Created File: 2253.json
Created File: 4049.json Created File: 4049.json
Created File: 1842.json Created File: 1842.json
Created File: 9924.json Created File: 9924.json
Created File: 987.json Created File: 987.json
Created File: 1044.json Created File: 1044.json
Created File: 3263.json Created File: 3263.json
Created File: 5338.json Created File: 5338.json
Created File: 6672.json Created File: 6672.json
Created File: 2463.json Created File: 2463.json
Created File: 6370.json Created File: 6370.json
Created File: 5694.json Created File: 5694.json
Created File: 8602.json Created File: 8602.json
Created File: 4811.json Created File: 4811.json
Created File: 8241.json Created File: 8241.json
Created File: 32.json Created File: 32.json
Created File: 8817.json Created File: 8817.json
Created File: 6147.json Created File: 6147.json
Created File: 1447.json Created File: 1447.json
Created File: 9672.json Created File: 9672.json
Created File: 5301.json Created File: 5301.json
Created File: 1128.json Created File: 1128.json
Created File: 5829.json Created File: 5829.json
Created File: 3117.json Created File: 3117.json
Created File: 1888.json Created File: 1888.json
Created File: 2406.json Created File: 2406.json
Created File: 4388.json Created File: 4388.json
Created File: 3648.json Created File: 3648.json
Created File: 7226.json Created File: 7226.json
Created File: 8796.json Created File: 8796.json
Created File: 2034.json Created File: 2034.json
Created File: 10015.json Created File: 10015.json
Created File: 1642.json Created File: 1642.json
Created File: 4551.json Created File: 4551.json
Created File: 1691.json Created File: 1691.json
Created File: 4741.json Created File: 4741.json
Created File: 8245.json Created File: 8245.json
Created File: 6492.json Created File: 6492.json
Created File: 10089.json Created File: 10089.json
Created File: 3853.json Created File: 3853.json
Created File: 1120.json Created File: 1120.json
Created File: 6187.json Created File: 6187.json
Created File: 4555.json Created File: 4555.json
Created File: 2496.json Created File: 2496.json
Created File: 5625.json Created File: 5625.json
Created File: 5414.json Created File: 5414.json
Created File: 5536.json Created File: 5536.json
Created File: 2110.json Created File: 2110.json
Created File: 7153.json Created File: 7153.json
Created File: 3148.json Created File: 3148.json
Created File: 3067.json Created File: 3067.json
Created File: 5833.json Created File: 5833.json
Created File: 2150.json Created File: 2150.json
Created File: 2981.json Created File: 2981.json
Created File: 5254.json Created File: 5254.json
Created File: 1250.json Created File: 1250.json
Created File: 6077.json Created File: 6077.json
Created File: 5866.json Created File: 5866.json
Created File: 6427.json Created File: 6427.json
Created File: 3745.json Created File: 3745.json
Created File: 6482.json Created File: 6482.json
Created File: 3145.json Created File: 3145.json
Created File: 5222.json Created File: 5222.json
Created File: 985.json Created File: 985.json
Created File: 6361.json Created File: 6361.json
Created File: 9040.json Created File: 9040.json
Created File: 9017.json Created File: 9017.json
Created File: 4951.json Created File: 4951.json
Created File: 6367.json Created File: 6367.json
Created File: 5464.json Created File: 5464.json
Created File: 8454.json Created File: 8454.json
Created File: 4924.json Created File: 4924.json
Created File: 1025.json Created File: 1025.json
Created File: 6330.json Created File: 6330.json
Created File: 5078.json Created File: 5078.json
Created File: 289.json Created File: 289.json
Created File: 6665.json Created File: 6665.json
Created File: 3986.json Created File: 3986.json
Created File: 1356.json Created File: 1356.json
Created File: 7883.json Created File: 7883.json
Created File: 3981.json Created File: 3981.json
Created File: 6145.json Created File: 6145.json
Created File: 3719.json Created File: 3719.json
Created File: 3255.json Created File: 3255.json
Created File: 953.json Created File: 953.json
Created File: 5775.json Created File: 5775.json
Created File: 4879.json Created File: 4879.json
Created File: 6023.json Created File: 6023.json
Created File: 2225.json Created File: 2225.json
Created File: 2019.json Created File: 2019.json
Created File: 3127.json Created File: 3127.json
Created File: 8717.json Created File: 8717.json
Created File: 5353.json Created File: 5353.json
Created File: 2518.json Created File: 2518.json
Created File: 5027.json Created File: 5027.json
Created File: 1947.json Created File: 1947.json
Created File: 1549.json Created File: 1549.json
Created File: 8832.json Created File: 8832.json
Created File: 8909.json Created File: 8909.json
Created File: 4628.json Created File: 4628.json
Created File: 2250.json Created File: 2250.json
Created File: 9797.json Created File: 9797.json
Created File: 3575.json Created File: 3575.json
Created File: 495.json Created File: 495.json
Created File: 5344.json Created File: 5344.json
Created File: 2245.json Created File: 2245.json
Created File: 6893.json Created File: 6893.json
Created File: 5530.json Created File: 5530.json
Created File: 7743.json Created File: 7743.json
Created File: 8626.json Created File: 8626.json
Created File: 4302.json Created File: 4302.json
Created File: 2202.json Created File: 2202.json
Created File: 5888.json Created File: 5888.json
Created File: 8654.json Created File: 8654.json
Created File: 9183.json Created File: 9183.json
Created File: 5787.json Created File: 5787.json
Created File: 2965.json Created File: 2965.json
Created File: 6324.json Created File: 6324.json
Created File: 5951.json Created File: 5951.json
Created File: 4161.json Created File: 4161.json
Created File: 9601.json Created File: 9601.json
Created File: 5676.json Created File: 5676.json
Created File: 5050.json Created File: 5050.json
Created File: 6585.json Created File: 6585.json
Created File: 8707.json Created File: 8707.json
Created File: 6860.json Created File: 6860.json
Created File: 679.json Created File: 679.json
Created File: 1288.json Created File: 1288.json
Created File: 9162.json Created File: 9162.json
Created File: 2611.json Created File: 2611.json
Created File: 4636.json Created File: 4636.json
Created File: 3817.json Created File: 3817.json
Created File: 1629.json Created File: 1629.json
Created File: 7369.json Created File: 7369.json
Created File: 2952.json Created File: 2952.json
Created File: 4204.json Created File: 4204.json
Created File: 8749.json Created File: 8749.json
Created File: 5786.json Created File: 5786.json
Created File: 8127.json Created File: 8127.json
Created File: 5169.json Created File: 5169.json
Created File: 2044.json Created File: 2044.json
Created File: 8519.json Created File: 8519.json
Created File: 2546.json Created File: 2546.json
Created File: 4424.json Created File: 4424.json
Created File: 9693.json Created File: 9693.json
Created File: 1392.json Created File: 1392.json
Created File: 8566.json Created File: 8566.json
Created File: 5946.json Created File: 5946.json
Created File: 6397.json Created File: 6397.json
Created File: 7131.json Created File: 7131.json
Created File: 4498.json Created File: 4498.json
Created File: 6745.json Created File: 6745.json
Created File: 6065.json Created File: 6065.json
Created File: 5508.json Created File: 5508.json
Created File: 3475.json Created File: 3475.json
Created File: 5987.json Created File: 5987.json
Created File: 1368.json Created File: 1368.json
Created File: 8708.json Created File: 8708.json
Created File: 9290.json Created File: 9290.json
Created File: 5120.json Created File: 5120.json
Created File: 5274.json Created File: 5274.json
Created File: 8912.json Created File: 8912.json
Created File: 4852.json Created File: 4852.json
Created File: 1757.json Created File: 1757.json
Created File: 7260.json Created File: 7260.json
Created File: 6878.json Created File: 6878.json
Created File: 2173.json Created File: 2173.json
Created File: 4701.json Created File: 4701.json
Created File: 7353.json Created File: 7353.json
Created File: 7378.json Created File: 7378.json
Created File: 5696.json Created File: 5696.json
Created File: 6044.json Created File: 6044.json
Created File: 1641.json Created File: 1641.json
Created File: 3490.json Created File: 3490.json
Created File: 4292.json Created File: 4292.json
Created File: 2235.json Created File: 2235.json
Created File: 1696.json Created File: 1696.json
Created File: 6399.json Created File: 6399.json
Created File: 4257.json Created File: 4257.json
Created File: 5394.json Created File: 5394.json
Created File: 1956.json Created File: 1956.json
Created File: 2461.json Created File: 2461.json
Created File: 5679.json Created File: 5679.json
Created File: 7159.json Created File: 7159.json
Created File: 5765.json Created File: 5765.json
Created File: 1389.json Created File: 1389.json
Created File: 7318.json Created File: 7318.json
Created File: 2246.json Created File: 2246.json
Created File: 1743.json Created File: 1743.json
Created File: 8827.json Created File: 8827.json
Created File: 5663.json Created File: 5663.json
Created File: 1415.json Created File: 1415.json
Created File: 4220.json Created File: 4220.json
Created File: 2097.json Created File: 2097.json
Created File: 8089.json Created File: 8089.json
Created File: 8431.json Created File: 8431.json
Created File: 2141.json Created File: 2141.json
Created File: 7719.json Created File: 7719.json
Created File: 8947.json Created File: 8947.json
Created File: 3336.json Created File: 3336.json
Created File: 3047.json Created File: 3047.json
Created File: 1624.json Created File: 1624.json
Created File: 2066.json Created File: 2066.json
Created File: 499.json Created File: 499.json
Created File: 1439.json Created File: 1439.json
Created File: 348.json Created File: 348.json
Created File: 10006.json Created File: 10006.json
Created File: 7513.json Created File: 7513.json
Created File: 1469.json Created File: 1469.json
Created File: 6266.json Created File: 6266.json
Created File: 6851.json Created File: 6851.json
Created File: 6076.json Created File: 6076.json
Created File: 5263.json Created File: 5263.json
Created File: 3421.json Created File: 3421.json
Created File: 4872.json Created File: 4872.json
Created File: 1770.json Created File: 1770.json
Created File: 5479.json Created File: 5479.json
Created File: 6418.json Created File: 6418.json
Created File: 1213.json Created File: 1213.json
Created File: 553.json Created File: 553.json
Created File: 9737.json Created File: 9737.json
Created File: 9114.json Created File: 9114.json
Created File: 6074.json Created File: 6074.json
Created File: 8812.json Created File: 8812.json
Created File: 3705.json Created File: 3705.json
Created File: 1493.json Created File: 1493.json
Created File: 8415.json Created File: 8415.json
Created File: 6184.json Created File: 6184.json
Created File: 1673.json Created File: 1673.json
Created File: 4514.json Created File: 4514.json
Created File: 6235.json Created File: 6235.json
Created File: 8701.json Created File: 8701.json
Created File: 4418.json Created File: 4418.json
Created File: 3357.json Created File: 3357.json
Created File: 8905.json Created File: 8905.json
Created File: 5228.json Created File: 5228.json
Created File: 7427.json Created File: 7427.json
Created File: 7386.json Created File: 7386.json
Created File: 5876.json Created File: 5876.json
Created File: 7003.json Created File: 7003.json
Created File: 9149.json Created File: 9149.json
Created File: 9240.json Created File: 9240.json
Created File: 2772.json Created File: 2772.json
Created File: 2879.json Created File: 2879.json
Created File: 2522.json Created File: 2522.json
Created File: 5235.json Created File: 5235.json
Created File: 4583.json Created File: 4583.json
Created File: 4338.json Created File: 4338.json
Created File: 1997.json Created File: 1997.json
Created File: 7360.json Created File: 7360.json
Created File: 800.json Created File: 800.json
Created File: 5234.json Created File: 5234.json
Created File: 8854.json Created File: 8854.json
Created File: 6682.json Created File: 6682.json
Created File: 6563.json Created File: 6563.json
Created File: 9084.json Created File: 9084.json
Created File: 15.json Created File: 15.json
Created File: 4950.json Created File: 4950.json
Created File: 5276.json Created File: 5276.json
Created File: 3238.json Created File: 3238.json
Created File: 6430.json Created File: 6430.json
Created File: 3556.json Created File: 3556.json
Created File: 2808.json Created File: 2808.json
Created File: 3211.json Created File: 3211.json
Created File: 1423.json Created File: 1423.json
Created File: 4035.json Created File: 4035.json
Created File: 7397.json Created File: 7397.json
Created File: 2860.json Created File: 2860.json
Created File: 5760.json Created File: 5760.json
Created File: 5181.json Created File: 5181.json
Created File: 7756.json Created File: 7756.json
Created File: 3316.json Created File: 3316.json
Created File: 8926.json Created File: 8926.json
Created File: 1223.json Created File: 1223.json
Created File: 8112.json Created File: 8112.json
Created File: 6736.json Created File: 6736.json
Created File: 8964.json Created File: 8964.json
Created File: 3364.json Created File: 3364.json
Created File: 8898.json Created File: 8898.json
Created File: 1839.json Created File: 1839.json
Created File: 3993.json Created File: 3993.json
Created File: 1960.json Created File: 1960.json
Created File: 8625.json Created File: 8625.json
Created File: 3130.json Created File: 3130.json
Created File: 4913.json Created File: 4913.json
Created File: 1672.json Created File: 1672.json
Created File: 7200.json Created File: 7200.json
Created File: 7438.json Created File: 7438.json
Created File: 284.json Created File: 284.json
Created File: 4437.json Created File: 4437.json
Created File: 4439.json Created File: 4439.json
Created File: 8956.json Created File: 8956.json
Created File: 3911.json Created File: 3911.json
Created File: 8611.json Created File: 8611.json
Created File: 8793.json Created File: 8793.json
Created File: 1446.json Created File: 1446.json
Created File: 10135.json Created File: 10135.json
Created File: 4268.json Created File: 4268.json
Created File: 10023.json Created File: 10023.json
Created File: 8676.json Created File: 8676.json
Created File: 7154.json Created File: 7154.json
Created File: 7338.json Created File: 7338.json
Created File: 971.json Created File: 971.json
Created File: 3205.json Created File: 3205.json
Created File: 9840.json Created File: 9840.json
Created File: 5635.json Created File: 5635.json
Created File: 3028.json Created File: 3028.json
Created File: 8189.json Created File: 8189.json
Created File: 1566.json Created File: 1566.json
Created File: 9165.json Created File: 9165.json
Created File: 9728.json Created File: 9728.json
Created File: 3033.json Created File: 3033.json
Created File: 1136.json Created File: 1136.json
Created File: 1411.json Created File: 1411.json
Created File: 8914.json Created File: 8914.json
Created File: 6695.json Created File: 6695.json
Created File: 2887.json Created File: 2887.json
Created File: 7924.json Created File: 7924.json
Created File: 2416.json Created File: 2416.json
Created File: 3431.json Created File: 3431.json
Created File: 1708.json Created File: 1708.json
Created File: 5071.json Created File: 5071.json
Created File: 3569.json Created File: 3569.json
Created File: 5142.json Created File: 5142.json
Created File: 2190.json Created File: 2190.json
Created File: 8891.json Created File: 8891.json
Created File: 1198.json Created File: 1198.json
Created File: 6543.json Created File: 6543.json
Created File: 3587.json Created File: 3587.json
Created File: 5260.json Created File: 5260.json
Created File: 3491.json Created File: 3491.json
Created File: 4483.json Created File: 4483.json
Created File: 3081.json Created File: 3081.json
Created File: 8867.json Created File: 8867.json
Created File: 6363.json Created File: 6363.json
Created File: 65.json Created File: 65.json
Created File: 2119.json Created File: 2119.json
Created File: 4899.json Created File: 4899.json
Created File: 1275.json Created File: 1275.json
Created File: 4293.json Created File: 4293.json
Created File: 2130.json Created File: 2130.json
Created File: 1305.json Created File: 1305.json
Created File: 3376.json Created File: 3376.json
Created File: 4433.json Created File: 4433.json
Created File: 5921.json Created File: 5921.json
Created File: 6992.json Created File: 6992.json
Created File: 3480.json Created File: 3480.json
Created File: 5170.json Created File: 5170.json
Created File: 10117.json Created File: 10117.json
Created File: 3959.json Created File: 3959.json
Created File: 8660.json Created File: 8660.json
Created File: 1326.json Created File: 1326.json
Created File: 2823.json Created File: 2823.json
Created File: 924.json Created File: 924.json
Created File: 6263.json Created File: 6263.json
Created File: 3643.json Created File: 3643.json
Created File: 6143.json Created File: 6143.json
Created File: 8395.json Created File: 8395.json
Created File: 8705.json Created File: 8705.json
Created File: 8879.json Created File: 8879.json
Created File: 7013.json Created File: 7013.json
Created File: 1649.json Created File: 1649.json
Created File: 2843.json Created File: 2843.json
Created File: 6681.json Created File: 6681.json
Created File: 4672.json Created File: 4672.json
Created File: 5328.json Created File: 5328.json
Created File: 3522.json Created File: 3522.json
Created File: 8858.json Created File: 8858.json
Created File: 4973.json Created File: 4973.json
Created File: 6204.json Created File: 6204.json
Created File: 5544.json Created File: 5544.json
Created File: 1383.json Created File: 1383.json
Created File: 9178.json Created File: 9178.json
Created File: 3022.json Created File: 3022.json
Created File: 4210.json Created File: 4210.json
Created File: 5703.json Created File: 5703.json
Created File: 4393.json Created File: 4393.json
Created File: 5879.json Created File: 5879.json
Created File: 4163.json Created File: 4163.json
Created File: 3440.json Created File: 3440.json
Created File: 5938.json Created File: 5938.json
Created File: 4687.json Created File: 4687.json
Created File: 1938.json Created File: 1938.json
Created File: 4876.json Created File: 4876.json
Created File: 9001.json Created File: 9001.json
Created File: 9915.json Created File: 9915.json
Created File: 6558.json Created File: 6558.json
Created File: 2603.json Created File: 2603.json
Created File: 3175.json Created File: 3175.json
Created File: 10029.json Created File: 10029.json
Created File: 7320.json Created File: 7320.json
Created File: 3324.json Created File: 3324.json
Created File: 10044.json Created File: 10044.json
Created File: 5195.json Created File: 5195.json
Created File: 1322.json Created File: 1322.json
Created File: 2846.json Created File: 2846.json
Created File: 10090.json Created File: 10090.json
Created File: 7395.json Created File: 7395.json
Created File: 3279.json Created File: 3279.json
Created File: 4460.json Created File: 4460.json
Created File: 4972.json Created File: 4972.json
Created File: 5932.json Created File: 5932.json
Created File: 3730.json Created File: 3730.json
Created File: 7499.json Created File: 7499.json
Created File: 1137.json Created File: 1137.json
Created File: 6449.json Created File: 6449.json
Created File: 2810.json Created File: 2810.json
Created File: 7201.json Created File: 7201.json
Created File: 600.json Created File: 600.json
Created File: 2483.json Created File: 2483.json
Created File: 2930.json Created File: 2930.json
Created File: 6354.json Created File: 6354.json
Created File: 2011.json Created File: 2011.json
Created File: 5706.json Created File: 5706.json
Created File: 8783.json Created File: 8783.json
Created File: 3389.json Created File: 3389.json
Created File: 4304.json Created File: 4304.json
Created File: 6108.json Created File: 6108.json
Created File: 10082.json Created File: 10082.json
Created File: 10176.json Created File: 10176.json
Created File: 5446.json Created File: 5446.json
Created File: 4710.json Created File: 4710.json
Created File: 363.json Created File: 363.json
Created File: 9902.json Created File: 9902.json
Created File: 2892.json Created File: 2892.json
Created File: 7350.json Created File: 7350.json
Created File: 6987.json Created File: 6987.json
Created File: 59.json Created File: 59.json
Created File: 5256.json Created File: 5256.json
Created File: 1023.json Created File: 1023.json
Created File: 5334.json Created File: 5334.json
Created File: 5218.json Created File: 5218.json
Created File: 2031.json Created File: 2031.json
Created File: 7632.json Created File: 7632.json
Created File: 8621.json Created File: 8621.json
Created File: 313.json Created File: 313.json
Created File: 4441.json Created File: 4441.json
Created File: 9967.json Created File: 9967.json
Created File: 8247.json Created File: 8247.json
Created File: 7073.json Created File: 7073.json
Created File: 9177.json Created File: 9177.json
Created File: 5903.json Created File: 5903.json
Created File: 487.json Created File: 487.json
Created File: 2015.json Created File: 2015.json
Created File: 1542.json Created File: 1542.json
Created File: 5072.json Created File: 5072.json
Created File: 9106.json Created File: 9106.json
Created File: 4889.json Created File: 4889.json
Created File: 1231.json Created File: 1231.json
Created File: 2259.json Created File: 2259.json
Created File: 6575.json Created File: 6575.json
Created File: 5127.json Created File: 5127.json
Created File: 4102.json Created File: 4102.json
Created File: 1339.json Created File: 1339.json
Created File: 10059.json Created File: 10059.json
Created File: 4511.json Created File: 4511.json
Created File: 6386.json Created File: 6386.json
Created File: 4842.json Created File: 4842.json
Created File: 4247.json Created File: 4247.json
Created File: 5520.json Created File: 5520.json
Created File: 5047.json Created File: 5047.json
Created File: 5415.json Created File: 5415.json
Created File: 9720.json Created File: 9720.json
Created File: 8770.json Created File: 8770.json
Created File: 4181.json Created File: 4181.json
Created File: 8946.json Created File: 8946.json
Created File: 2945.json Created File: 2945.json
Created File: 9787.json Created File: 9787.json
Created File: 9156.json Created File: 9156.json
Created File: 1871.json Created File: 1871.json
Created File: 7426.json Created File: 7426.json
Created File: 3227.json Created File: 3227.json
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
from decentralizepy.datasets.Femnist import Femnist from decentralizepy.datasets.Femnist import Femnist
f = Femnist() f = Femnist()
f.file_per_user('../leaf/data/femnist/data/train','../leaf/data/femnist/per_user_data/train') f.file_per_user('../leaf/data/femnist/data/train','../leaf/data/femnist/per_user_data/train')
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
a = set() a = set()
a.update([2, 3, 4, 5]) a.update([2, 3, 4, 5])
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
a a
``` ```
%% Output %% Output
{2, 3, 4, 5} {2, 3, 4, 5}
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
print(*a) print(*a)
``` ```
%% Output %% Output
2 3 4 5 2 3 4 5
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
from decentralizepy.graphs.FullyConnected import FullyConnected from decentralizepy.graphs.FullyConnected import FullyConnected
s = FullyConnected(96) s = FullyConnected(96)
s.write_graph_to_file('96_node_fullyConnected.edges') s.write_graph_to_file('96_node_fullyConnected.edges')
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
from decentralizepy.graphs.SmallWorld import SmallWorld from decentralizepy.graphs.SmallWorld import SmallWorld
s1 = SmallWorld(96, 2, 1.0) s1 = SmallWorld(96, 2, 1.0)
s1.write_graph_to_file('96_nodes_random1.edges') s1.write_graph_to_file('96_nodes_random1.edges')
s2 = SmallWorld(96, 2, 1.0) s2 = SmallWorld(96, 2, 1.0)
s2.write_graph_to_file('96_nodes_random2.edges') s2.write_graph_to_file('96_nodes_random2.edges')
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
import sys import sys
sys.argv sys.argv
``` ```
%% Output %% Output
['/home/risharma/miniconda3/envs/decpy/lib/python3.9/site-packages/ipykernel_launcher.py', ['/home/risharma/miniconda3/envs/decpy/lib/python3.9/site-packages/ipykernel_launcher.py',
'--ip=127.0.0.1', '--ip=127.0.0.1',
'--stdin=9008', '--stdin=9008',
'--control=9006', '--control=9006',
'--hb=9005', '--hb=9005',
'--Session.signature_scheme="hmac-sha256"', '--Session.signature_scheme="hmac-sha256"',
'--Session.key=b"eac5d2f8-c460-45f1-a268-1e4b46a6efd6"', '--Session.key=b"eac5d2f8-c460-45f1-a268-1e4b46a6efd6"',
'--shell=9007', '--shell=9007',
'--transport="tcp"', '--transport="tcp"',
'--iopub=9009', '--iopub=9009',
'--f=/tmp/tmp-21212479paJaUBJBN84.json'] '--f=/tmp/tmp-21212479paJaUBJBN84.json']
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
import torch import torch
from decentralizepy.datasets.Femnist import CNN from decentralizepy.datasets.Femnist import CNN
m1 = CNN() m1 = CNN()
o1 = torch.optim.SGD(m1.parameters(), 0.6) o1 = torch.optim.SGD(m1.parameters(), 0.6)
#print("m1_parameters: ", {k:v.data for k, v in zip(m1.state_dict(), m1.parameters())}) #print("m1_parameters: ", {k:v.data for k, v in zip(m1.state_dict(), m1.parameters())})
#print("m1_state_dict: ", m1.state_dict()) #print("m1_state_dict: ", m1.state_dict())
#print("o1_state_dict: ", o1.state_dict()) #print("o1_state_dict: ", o1.state_dict())
tensors_to_cat = [v.data for _, v in m1.state_dict().items()] tensors_to_cat = [v.data for _, v in m1.state_dict().items()]
print("tensors to cat: ", tensors_to_cat) print("tensors to cat: ", tensors_to_cat)
``` ```
%% Output %% Output
tensors to cat: [tensor([[[[-0.0754, -0.1052, 0.0336, 0.0976, -0.1882], tensors to cat: [tensor([[[[-0.0754, -0.1052, 0.0336, 0.0976, -0.1882],
[ 0.1723, 0.0860, -0.0557, -0.1182, 0.1845], [ 0.1723, 0.0860, -0.0557, -0.1182, 0.1845],
[-0.1112, 0.0554, -0.0190, 0.1264, -0.0851], [-0.1112, 0.0554, -0.0190, 0.1264, -0.0851],
[-0.0788, 0.1281, -0.0477, 0.1910, 0.0089], [-0.0788, 0.1281, -0.0477, 0.1910, 0.0089],
[-0.1205, -0.0230, -0.0574, 0.1633, -0.1101]]], [-0.1205, -0.0230, -0.0574, 0.1633, -0.1101]]],
[[[ 0.0271, -0.1426, -0.0381, 0.1128, 0.0396], [[[ 0.0271, -0.1426, -0.0381, 0.1128, 0.0396],
[-0.0023, 0.1021, -0.0483, -0.0167, 0.0090], [-0.0023, 0.1021, -0.0483, -0.0167, 0.0090],
[-0.0645, -0.1863, -0.1494, -0.0473, 0.1572], [-0.0645, -0.1863, -0.1494, -0.0473, 0.1572],
[-0.0027, -0.0923, 0.0186, 0.0787, 0.1404], [-0.0027, -0.0923, 0.0186, 0.0787, 0.1404],
[ 0.0714, 0.0434, 0.1853, 0.1262, 0.0502]]], [ 0.0714, 0.0434, 0.1853, 0.1262, 0.0502]]],
[[[-0.1708, 0.0628, -0.1877, -0.0879, -0.1190], [[[-0.1708, 0.0628, -0.1877, -0.0879, -0.1190],
[-0.0111, 0.0684, -0.1659, -0.1297, 0.0684], [-0.0111, 0.0684, -0.1659, -0.1297, 0.0684],
[ 0.1629, -0.1196, -0.0096, 0.1854, -0.0721], [ 0.1629, -0.1196, -0.0096, 0.1854, -0.0721],
[ 0.1122, -0.0325, 0.1887, -0.0685, 0.1441], [ 0.1122, -0.0325, 0.1887, -0.0685, 0.1441],
[ 0.1734, 0.0720, -0.0941, -0.0536, 0.0355]]], [ 0.1734, 0.0720, -0.0941, -0.0536, 0.0355]]],
[[[-0.1465, -0.0899, 0.1122, 0.0323, -0.1287], [[[-0.1465, -0.0899, 0.1122, 0.0323, -0.1287],
[-0.1237, 0.0852, -0.1984, -0.0995, 0.0734], [-0.1237, 0.0852, -0.1984, -0.0995, 0.0734],
[ 0.0521, -0.1664, -0.0875, 0.0643, -0.1166], [ 0.0521, -0.1664, -0.0875, 0.0643, -0.1166],
[ 0.1862, 0.1754, -0.0282, 0.1074, -0.1337], [ 0.1862, 0.1754, -0.0282, 0.1074, -0.1337],
[ 0.1779, 0.1883, 0.1376, -0.1518, -0.1619]]], [ 0.1779, 0.1883, 0.1376, -0.1518, -0.1619]]],
[[[ 0.0158, -0.1902, 0.0607, 0.1062, 0.0373], [[[ 0.0158, -0.1902, 0.0607, 0.1062, 0.0373],
[-0.1737, -0.1112, 0.0597, -0.1030, -0.0897], [-0.1737, -0.1112, 0.0597, -0.1030, -0.0897],
[ 0.0892, 0.1166, -0.0127, 0.1859, -0.1859], [ 0.0892, 0.1166, -0.0127, 0.1859, -0.1859],
[ 0.1935, -0.1200, 0.0472, -0.0144, -0.0219], [ 0.1935, -0.1200, 0.0472, -0.0144, -0.0219],
[-0.0596, -0.0255, -0.1733, 0.1746, 0.1136]]], [-0.0596, -0.0255, -0.1733, 0.1746, 0.1136]]],
[[[-0.1769, 0.0885, 0.0706, 0.1742, 0.1178], [[[-0.1769, 0.0885, 0.0706, 0.1742, 0.1178],
[-0.1831, 0.0380, -0.0769, -0.1068, 0.1958], [-0.1831, 0.0380, -0.0769, -0.1068, 0.1958],
[-0.0166, -0.0059, 0.0663, 0.0610, 0.0321], [-0.0166, -0.0059, 0.0663, 0.0610, 0.0321],
[-0.1393, -0.1884, -0.0678, -0.1400, 0.1042], [-0.1393, -0.1884, -0.0678, -0.1400, 0.1042],
[ 0.0653, -0.1733, 0.0744, 0.1278, 0.0482]]], [ 0.0653, -0.1733, 0.0744, 0.1278, 0.0482]]],
[[[ 0.0935, -0.0115, -0.1991, 0.1451, -0.1708], [[[ 0.0935, -0.0115, -0.1991, 0.1451, -0.1708],
[ 0.1979, -0.0709, 0.1771, 0.0486, 0.1292], [ 0.1979, -0.0709, 0.1771, 0.0486, 0.1292],
[-0.1624, 0.0661, -0.0004, -0.0079, 0.1195], [-0.1624, 0.0661, -0.0004, -0.0079, 0.1195],
[ 0.0655, 0.1849, 0.1179, 0.1520, -0.0723], [ 0.0655, 0.1849, 0.1179, 0.1520, -0.0723],
[ 0.1374, 0.1547, 0.1871, 0.1924, 0.0539]]], [ 0.1374, 0.1547, 0.1871, 0.1924, 0.0539]]],
[[[-0.1602, 0.1380, 0.1321, -0.0722, -0.0291], [[[-0.1602, 0.1380, 0.1321, -0.0722, -0.0291],
[ 0.0820, -0.0490, -0.1345, 0.1719, 0.1306], [ 0.0820, -0.0490, -0.1345, 0.1719, 0.1306],
[-0.0183, 0.0615, 0.1189, -0.0707, -0.0696], [-0.0183, 0.0615, 0.1189, -0.0707, -0.0696],
[ 0.0017, -0.1185, -0.0766, 0.1017, -0.0518], [ 0.0017, -0.1185, -0.0766, 0.1017, -0.0518],
[-0.1316, -0.1323, -0.0135, 0.0103, -0.1509]]], [-0.1316, -0.1323, -0.0135, 0.0103, -0.1509]]],
[[[-0.1189, -0.1300, -0.1908, -0.0222, 0.0017], [[[-0.1189, -0.1300, -0.1908, -0.0222, 0.0017],
[-0.0644, 0.0996, 0.0454, 0.0198, -0.1608], [-0.0644, 0.0996, 0.0454, 0.0198, -0.1608],
[-0.0614, -0.0721, -0.0223, -0.0746, -0.0081], [-0.0614, -0.0721, -0.0223, -0.0746, -0.0081],
[-0.0133, 0.1326, 0.1275, -0.0954, 0.0281], [-0.0133, 0.1326, 0.1275, -0.0954, 0.0281],
[ 0.1004, -0.0435, -0.1027, 0.1867, 0.0034]]], [ 0.1004, -0.0435, -0.1027, 0.1867, 0.0034]]],
[[[-0.0022, -0.0830, -0.0399, 0.1039, -0.1230], [[[-0.0022, -0.0830, -0.0399, 0.1039, -0.1230],
[-0.0067, 0.1032, 0.0895, 0.0679, 0.0133], [-0.0067, 0.1032, 0.0895, 0.0679, 0.0133],
[ 0.0148, -0.1275, 0.0391, -0.0173, 0.1621], [ 0.0148, -0.1275, 0.0391, -0.0173, 0.1621],
[ 0.0532, -0.1873, -0.1499, 0.0605, -0.0016], [ 0.0532, -0.1873, -0.1499, 0.0605, -0.0016],
[ 0.0386, -0.0360, 0.0255, -0.0580, -0.1954]]], [ 0.0386, -0.0360, 0.0255, -0.0580, -0.1954]]],
[[[-0.0186, -0.1922, -0.1356, 0.1696, 0.0111], [[[-0.0186, -0.1922, -0.1356, 0.1696, 0.0111],
[ 0.1121, -0.0990, 0.0789, -0.0053, 0.0729], [ 0.1121, -0.0990, 0.0789, -0.0053, 0.0729],
[-0.0565, -0.0405, 0.0559, 0.1065, -0.1723], [-0.0565, -0.0405, 0.0559, 0.1065, -0.1723],
[ 0.0569, -0.0432, -0.0484, 0.1533, 0.0421], [ 0.0569, -0.0432, -0.0484, 0.1533, 0.0421],
[-0.0369, 0.1050, 0.1623, -0.1503, 0.0680]]], [-0.0369, 0.1050, 0.1623, -0.1503, 0.0680]]],
[[[-0.0028, -0.0167, 0.0598, 0.0854, -0.1996], [[[-0.0028, -0.0167, 0.0598, 0.0854, -0.1996],
[ 0.0548, -0.0586, 0.1987, -0.0415, -0.0477], [ 0.0548, -0.0586, 0.1987, -0.0415, -0.0477],
[ 0.1172, -0.0924, 0.1351, 0.1319, 0.0983], [ 0.1172, -0.0924, 0.1351, 0.1319, 0.0983],
[ 0.0756, 0.1345, 0.1360, -0.0779, -0.1169], [ 0.0756, 0.1345, 0.1360, -0.0779, -0.1169],
[ 0.1524, 0.0208, 0.0672, 0.1320, 0.1958]]], [ 0.1524, 0.0208, 0.0672, 0.1320, 0.1958]]],
[[[-0.0794, -0.1947, 0.0410, 0.1480, -0.0168], [[[-0.0794, -0.1947, 0.0410, 0.1480, -0.0168],
[ 0.0836, -0.0750, -0.0903, -0.1946, 0.0676], [ 0.0836, -0.0750, -0.0903, -0.1946, 0.0676],
[-0.1202, 0.0276, 0.0058, -0.1465, 0.1545], [-0.1202, 0.0276, 0.0058, -0.1465, 0.1545],
[-0.1914, 0.0926, -0.0799, -0.0715, -0.0607], [-0.1914, 0.0926, -0.0799, -0.0715, -0.0607],
[ 0.1624, 0.1929, -0.1532, -0.0007, -0.1312]]], [ 0.1624, 0.1929, -0.1532, -0.0007, -0.1312]]],
[[[ 0.0281, -0.0407, -0.0482, 0.0492, 0.0039], [[[ 0.0281, -0.0407, -0.0482, 0.0492, 0.0039],
[-0.0752, -0.0298, 0.0764, -0.0738, -0.0705], [-0.0752, -0.0298, 0.0764, -0.0738, -0.0705],
[ 0.1404, -0.1254, 0.1613, -0.0501, -0.1432], [ 0.1404, -0.1254, 0.1613, -0.0501, -0.1432],
[ 0.0132, -0.0170, -0.0971, -0.0117, 0.1406], [ 0.0132, -0.0170, -0.0971, -0.0117, 0.1406],
[ 0.0655, -0.1970, 0.1782, 0.0665, 0.1174]]], [ 0.0655, -0.1970, 0.1782, 0.0665, 0.1174]]],
[[[ 0.1879, 0.0858, 0.0340, 0.1138, -0.0841], [[[ 0.1879, 0.0858, 0.0340, 0.1138, -0.0841],
[ 0.1503, -0.1680, 0.0618, 0.1971, -0.0156], [ 0.1503, -0.1680, 0.0618, 0.1971, -0.0156],
[-0.1211, -0.0485, 0.0675, -0.0536, 0.0046], [-0.1211, -0.0485, 0.0675, -0.0536, 0.0046],
[ 0.1134, 0.1310, -0.0966, -0.1924, -0.0812], [ 0.1134, 0.1310, -0.0966, -0.1924, -0.0812],
[-0.0800, -0.0553, 0.1818, -0.0217, 0.0107]]], [-0.0800, -0.0553, 0.1818, -0.0217, 0.0107]]],
[[[ 0.1118, 0.1368, 0.0269, -0.0690, 0.0168], [[[ 0.1118, 0.1368, 0.0269, -0.0690, 0.0168],
[-0.1371, -0.1983, 0.0016, -0.0094, 0.0502], [-0.1371, -0.1983, 0.0016, -0.0094, 0.0502],
[-0.0219, 0.1939, -0.1821, 0.0988, -0.1016], [-0.0219, 0.1939, -0.1821, 0.0988, -0.1016],
[-0.0063, 0.0789, 0.1037, -0.0646, -0.1694], [-0.0063, 0.0789, 0.1037, -0.0646, -0.1694],
[-0.0464, 0.0624, 0.0681, -0.1855, 0.1219]]], [-0.0464, 0.0624, 0.0681, -0.1855, 0.1219]]],
[[[ 0.0640, 0.1420, -0.0144, -0.0606, 0.0963], [[[ 0.0640, 0.1420, -0.0144, -0.0606, 0.0963],
[ 0.1349, 0.0906, 0.0826, -0.0385, -0.1635], [ 0.1349, 0.0906, 0.0826, -0.0385, -0.1635],
[ 0.0278, 0.1875, 0.1559, 0.1424, 0.0878], [ 0.0278, 0.1875, 0.1559, 0.1424, 0.0878],
[-0.0593, -0.0847, -0.0458, 0.1402, 0.0160], [-0.0593, -0.0847, -0.0458, 0.1402, 0.0160],
[ 0.1887, -0.1220, -0.0316, -0.0892, -0.0401]]], [ 0.1887, -0.1220, -0.0316, -0.0892, -0.0401]]],
[[[-0.1103, -0.0884, 0.0775, -0.0338, -0.1245], [[[-0.1103, -0.0884, 0.0775, -0.0338, -0.1245],
[-0.1848, 0.0657, 0.0945, 0.0677, -0.1089], [-0.1848, 0.0657, 0.0945, 0.0677, -0.1089],
[ 0.0585, -0.0791, -0.0535, 0.0527, -0.0741], [ 0.0585, -0.0791, -0.0535, 0.0527, -0.0741],
[ 0.1146, 0.1908, -0.0167, -0.0033, -0.1418], [ 0.1146, 0.1908, -0.0167, -0.0033, -0.1418],
[-0.1011, 0.0829, -0.1472, -0.1310, -0.0991]]], [-0.1011, 0.0829, -0.1472, -0.1310, -0.0991]]],
[[[ 0.0865, 0.1608, 0.0168, -0.1126, -0.0889], [[[ 0.0865, 0.1608, 0.0168, -0.1126, -0.0889],
[-0.1322, 0.1052, 0.1285, 0.0574, 0.1983], [-0.1322, 0.1052, 0.1285, 0.0574, 0.1983],
[ 0.0788, 0.0125, -0.0781, -0.0303, -0.1089], [ 0.0788, 0.0125, -0.0781, -0.0303, -0.1089],
[-0.1860, -0.1878, 0.0608, 0.0402, -0.0390], [-0.1860, -0.1878, 0.0608, 0.0402, -0.0390],
[-0.1797, -0.1828, 0.0475, -0.1754, 0.0657]]], [-0.1797, -0.1828, 0.0475, -0.1754, 0.0657]]],
[[[-0.0632, 0.1015, -0.1493, -0.0855, 0.1822], [[[-0.0632, 0.1015, -0.1493, -0.0855, 0.1822],
[-0.1492, 0.0236, -0.1418, -0.1826, -0.1978], [-0.1492, 0.0236, -0.1418, -0.1826, -0.1978],
[ 0.0854, -0.1503, 0.1953, -0.0668, -0.1042], [ 0.0854, -0.1503, 0.1953, -0.0668, -0.1042],
[ 0.1537, -0.0573, 0.0178, 0.1350, 0.1823], [ 0.1537, -0.0573, 0.0178, 0.1350, 0.1823],
[-0.1131, -0.0838, -0.1445, -0.1476, -0.0288]]], [-0.1131, -0.0838, -0.1445, -0.1476, -0.0288]]],
[[[ 0.1445, 0.1748, 0.0146, -0.0807, -0.1905], [[[ 0.1445, 0.1748, 0.0146, -0.0807, -0.1905],
[-0.1474, -0.0938, 0.0965, 0.0233, 0.0757], [-0.1474, -0.0938, 0.0965, 0.0233, 0.0757],
[-0.1257, -0.1316, -0.0105, 0.1447, -0.1200], [-0.1257, -0.1316, -0.0105, 0.1447, -0.1200],
[ 0.0921, -0.0332, -0.1153, -0.0260, 0.0640], [ 0.0921, -0.0332, -0.1153, -0.0260, 0.0640],
[-0.1528, 0.1237, 0.1956, 0.0167, 0.0205]]], [-0.1528, 0.1237, 0.1956, 0.0167, 0.0205]]],
[[[-0.0236, -0.1264, -0.0342, -0.1199, -0.1381], [[[-0.0236, -0.1264, -0.0342, -0.1199, -0.1381],
[-0.0895, -0.0116, -0.1108, 0.0994, 0.0744], [-0.0895, -0.0116, -0.1108, 0.0994, 0.0744],
[-0.1190, -0.1275, -0.1097, 0.0678, -0.1547], [-0.1190, -0.1275, -0.1097, 0.0678, -0.1547],
[-0.1405, 0.0500, -0.1195, 0.0917, -0.0952], [-0.1405, 0.0500, -0.1195, 0.0917, -0.0952],
[-0.1799, -0.1516, 0.0956, -0.0402, 0.0426]]], [-0.1799, -0.1516, 0.0956, -0.0402, 0.0426]]],
[[[ 0.0556, 0.1145, 0.0388, 0.0917, -0.0881], [[[ 0.0556, 0.1145, 0.0388, 0.0917, -0.0881],
[-0.0933, 0.0580, 0.1433, -0.1127, -0.0142], [-0.0933, 0.0580, 0.1433, -0.1127, -0.0142],
[-0.0014, 0.0792, -0.0451, -0.0696, 0.0486], [-0.0014, 0.0792, -0.0451, -0.0696, 0.0486],
[ 0.0006, 0.0264, -0.1252, 0.0796, 0.0561], [ 0.0006, 0.0264, -0.1252, 0.0796, 0.0561],
[-0.0203, -0.0664, -0.0443, 0.1173, -0.0193]]], [-0.0203, -0.0664, -0.0443, 0.1173, -0.0193]]],
[[[-0.0796, -0.1144, -0.0728, 0.0867, -0.1342], [[[-0.0796, -0.1144, -0.0728, 0.0867, -0.1342],
[-0.0150, -0.0840, -0.0134, 0.0634, 0.1743], [-0.0150, -0.0840, -0.0134, 0.0634, 0.1743],
[-0.0773, -0.0827, -0.0470, -0.0879, 0.0346], [-0.0773, -0.0827, -0.0470, -0.0879, 0.0346],
[ 0.0642, 0.1078, 0.0373, 0.1683, 0.0433], [ 0.0642, 0.1078, 0.0373, 0.1683, 0.0433],
[-0.0024, 0.1508, -0.1980, 0.0808, -0.1028]]], [-0.0024, 0.1508, -0.1980, 0.0808, -0.1028]]],
[[[-0.1358, -0.1896, -0.1495, 0.1018, 0.0048], [[[-0.1358, -0.1896, -0.1495, 0.1018, 0.0048],
[-0.1030, -0.1340, 0.0090, -0.0781, 0.0415], [-0.1030, -0.1340, 0.0090, -0.0781, 0.0415],
[-0.1069, 0.0658, -0.0144, 0.1455, 0.1006], [-0.1069, 0.0658, -0.0144, 0.1455, 0.1006],
[-0.0209, 0.0757, -0.1481, -0.0273, -0.0880], [-0.0209, 0.0757, -0.1481, -0.0273, -0.0880],
[ 0.0859, -0.1996, -0.1787, -0.1869, -0.0310]]], [ 0.0859, -0.1996, -0.1787, -0.1869, -0.0310]]],
[[[-0.1005, 0.1260, -0.1949, 0.1433, 0.0845], [[[-0.1005, 0.1260, -0.1949, 0.1433, 0.0845],
[ 0.0336, 0.1110, -0.0389, 0.0584, -0.0470], [ 0.0336, 0.1110, -0.0389, 0.0584, -0.0470],
[ 0.0333, 0.0861, -0.0371, 0.1471, -0.0385], [ 0.0333, 0.0861, -0.0371, 0.1471, -0.0385],
[ 0.1889, 0.1999, 0.1643, 0.0767, -0.1738], [ 0.1889, 0.1999, 0.1643, 0.0767, -0.1738],
[-0.1423, -0.0945, -0.0881, 0.0847, -0.0170]]], [-0.1423, -0.0945, -0.0881, 0.0847, -0.0170]]],
[[[-0.1417, 0.0120, -0.1466, -0.1825, -0.1018], [[[-0.1417, 0.0120, -0.1466, -0.1825, -0.1018],
[ 0.1989, -0.1077, -0.1263, -0.0141, 0.0296], [ 0.1989, -0.1077, -0.1263, -0.0141, 0.0296],
[ 0.0414, 0.1355, -0.1630, -0.1348, 0.1382], [ 0.0414, 0.1355, -0.1630, -0.1348, 0.1382],
[ 0.0589, 0.1278, 0.0218, 0.1217, 0.1369], [ 0.0589, 0.1278, 0.0218, 0.1217, 0.1369],
[ 0.0659, 0.1200, 0.0031, 0.0240, -0.1200]]], [ 0.0659, 0.1200, 0.0031, 0.0240, -0.1200]]],
[[[ 0.1293, -0.1227, -0.0351, -0.1777, 0.0515], [[[ 0.1293, -0.1227, -0.0351, -0.1777, 0.0515],
[-0.1087, 0.0541, 0.0684, 0.0929, -0.1989], [-0.1087, 0.0541, 0.0684, 0.0929, -0.1989],
[ 0.0596, -0.0794, -0.0461, 0.0260, -0.1812], [ 0.0596, -0.0794, -0.0461, 0.0260, -0.1812],
[-0.0437, 0.0989, -0.1171, -0.0531, -0.1493], [-0.0437, 0.0989, -0.1171, -0.0531, -0.1493],
[-0.1677, -0.1009, 0.0778, -0.1462, 0.0935]]], [-0.1677, -0.1009, 0.0778, -0.1462, 0.0935]]],
[[[-0.0794, 0.1744, -0.0915, -0.1289, -0.0317], [[[-0.0794, 0.1744, -0.0915, -0.1289, -0.0317],
[ 0.1679, 0.1361, 0.0542, 0.1967, 0.1620], [ 0.1679, 0.1361, 0.0542, 0.1967, 0.1620],
[-0.0898, 0.0305, -0.1617, -0.1643, -0.0253], [-0.0898, 0.0305, -0.1617, -0.1643, -0.0253],
[ 0.1339, -0.1151, 0.0954, -0.0359, -0.1790], [ 0.1339, -0.1151, 0.0954, -0.0359, -0.1790],
[-0.0408, -0.1074, 0.0081, -0.0884, 0.0163]]], [-0.0408, -0.1074, 0.0081, -0.0884, 0.0163]]],
[[[ 0.0022, -0.1097, 0.0842, -0.1927, 0.0277], [[[ 0.0022, -0.1097, 0.0842, -0.1927, 0.0277],
[ 0.0127, -0.0666, 0.1235, -0.1696, 0.1377], [ 0.0127, -0.0666, 0.1235, -0.1696, 0.1377],
[ 0.0741, -0.1627, -0.0953, 0.1622, 0.0921], [ 0.0741, -0.1627, -0.0953, 0.1622, 0.0921],
[ 0.1111, 0.1422, -0.1884, -0.0130, -0.0533], [ 0.1111, 0.1422, -0.1884, -0.0130, -0.0533],
[ 0.0838, 0.1769, 0.0109, 0.0086, -0.0795]]], [ 0.0838, 0.1769, 0.0109, 0.0086, -0.0795]]],
[[[-0.0502, -0.1545, -0.1304, 0.1680, -0.0439], [[[-0.0502, -0.1545, -0.1304, 0.1680, -0.0439],
[ 0.1891, 0.1681, -0.1068, 0.1589, -0.1011], [ 0.1891, 0.1681, -0.1068, 0.1589, -0.1011],
[-0.0923, -0.1569, 0.1337, 0.0743, 0.1794], [-0.0923, -0.1569, 0.1337, 0.0743, 0.1794],
[-0.1630, 0.0071, -0.0524, -0.1211, -0.0362], [-0.1630, 0.0071, -0.0524, -0.1211, -0.0362],
[-0.1111, 0.1168, 0.0010, -0.0352, 0.1756]]], [-0.1111, 0.1168, 0.0010, -0.0352, 0.1756]]],
[[[-0.0282, -0.0647, -0.0309, 0.0586, 0.1442], [[[-0.0282, -0.0647, -0.0309, 0.0586, 0.1442],
[ 0.1881, 0.1471, 0.0371, -0.0573, 0.1329], [ 0.1881, 0.1471, 0.0371, -0.0573, 0.1329],
[ 0.0760, -0.0789, -0.0935, 0.1464, 0.1953], [ 0.0760, -0.0789, -0.0935, 0.1464, 0.1953],
[-0.0965, -0.0576, 0.1569, -0.0305, 0.1716], [-0.0965, -0.0576, 0.1569, -0.0305, 0.1716],
[ 0.1946, 0.1921, 0.1460, 0.1270, 0.0192]]]]), tensor([ 0.1092, -0.1012, 0.1255, -0.0428, 0.1610, 0.0467, 0.1861, 0.1967, [ 0.1946, 0.1921, 0.1460, 0.1270, 0.0192]]]]), tensor([ 0.1092, -0.1012, 0.1255, -0.0428, 0.1610, 0.0467, 0.1861, 0.1967,
-0.0119, -0.0437, -0.0364, -0.1432, -0.1487, 0.1987, -0.0183, 0.0156, -0.0119, -0.0437, -0.0364, -0.1432, -0.1487, 0.1987, -0.0183, 0.0156,
0.0776, 0.1051, 0.1560, 0.1141, -0.1168, -0.0773, 0.1036, 0.0045, 0.0776, 0.1051, 0.1560, 0.1141, -0.1168, -0.0773, 0.1036, 0.0045,
0.0811, 0.0564, 0.1103, 0.1955, 0.1101, 0.1653, 0.1274, 0.0707]), tensor([[[[ 3.4215e-02, 3.1851e-02, -3.1990e-02, -2.4633e-02, -1.4061e-02], 0.0811, 0.0564, 0.1103, 0.1955, 0.1101, 0.1653, 0.1274, 0.0707]), tensor([[[[ 3.4215e-02, 3.1851e-02, -3.1990e-02, -2.4633e-02, -1.4061e-02],
[ 1.2874e-02, 1.9820e-02, -2.9326e-02, -3.5093e-02, -1.6611e-02], [ 1.2874e-02, 1.9820e-02, -2.9326e-02, -3.5093e-02, -1.6611e-02],
[-1.8654e-02, -2.6634e-02, -1.0565e-03, -3.2393e-02, 2.5394e-02], [-1.8654e-02, -2.6634e-02, -1.0565e-03, -3.2393e-02, 2.5394e-02],
[ 2.8582e-02, 2.3981e-02, -3.6160e-03, -3.3239e-02, -3.3089e-02], [ 2.8582e-02, 2.3981e-02, -3.6160e-03, -3.3239e-02, -3.3089e-02],
[-1.4720e-02, -1.4778e-02, -1.3619e-02, -8.4345e-03, 3.2735e-02]], [-1.4720e-02, -1.4778e-02, -1.3619e-02, -8.4345e-03, 3.2735e-02]],
[[-8.9284e-03, -1.1837e-02, -2.9223e-02, 2.7357e-02, 3.1966e-02], [[-8.9284e-03, -1.1837e-02, -2.9223e-02, 2.7357e-02, 3.1966e-02],
[-3.2790e-02, -2.8377e-02, 6.0738e-03, -1.0012e-02, 2.0444e-03], [-3.2790e-02, -2.8377e-02, 6.0738e-03, -1.0012e-02, 2.0444e-03],
[-2.6949e-02, 4.2564e-03, 2.1914e-02, -1.0865e-02, 6.7992e-03], [-2.6949e-02, 4.2564e-03, 2.1914e-02, -1.0865e-02, 6.7992e-03],
[-3.0702e-02, 3.4266e-02, -1.4731e-02, 5.6710e-03, -1.2800e-02], [-3.0702e-02, 3.4266e-02, -1.4731e-02, 5.6710e-03, -1.2800e-02],
[-2.6684e-02, -1.1684e-02, 2.0707e-02, 2.5899e-02, -2.3985e-02]], [-2.6684e-02, -1.1684e-02, 2.0707e-02, 2.5899e-02, -2.3985e-02]],
[[ 1.5901e-02, 2.6151e-02, 3.1089e-02, -1.1213e-02, 3.3065e-02], [[ 1.5901e-02, 2.6151e-02, 3.1089e-02, -1.1213e-02, 3.3065e-02],
[ 3.1703e-03, 2.5620e-02, 9.4295e-03, 2.3775e-02, -1.4995e-02], [ 3.1703e-03, 2.5620e-02, 9.4295e-03, 2.3775e-02, -1.4995e-02],
[ 2.9061e-02, -2.2070e-02, 1.8273e-02, -1.7573e-02, -8.4471e-03], [ 2.9061e-02, -2.2070e-02, 1.8273e-02, -1.7573e-02, -8.4471e-03],
[-2.3864e-02, -1.0588e-02, -1.6631e-02, 5.6322e-04, -3.7847e-03], [-2.3864e-02, -1.0588e-02, -1.6631e-02, 5.6322e-04, -3.7847e-03],
[-3.4452e-02, -4.9367e-03, -8.4581e-03, 9.4274e-03, 1.8410e-02]], [-3.4452e-02, -4.9367e-03, -8.4581e-03, 9.4274e-03, 1.8410e-02]],
..., ...,
[[ 1.8886e-02, -1.4010e-02, 3.1874e-02, 3.2005e-02, -2.3095e-02], [[ 1.8886e-02, -1.4010e-02, 3.1874e-02, 3.2005e-02, -2.3095e-02],
[-3.0070e-03, -7.3744e-04, 7.5279e-03, 3.4897e-03, 2.9886e-02], [-3.0070e-03, -7.3744e-04, 7.5279e-03, 3.4897e-03, 2.9886e-02],
[-7.0828e-03, -1.5942e-03, -2.5189e-02, 3.1235e-02, 1.0274e-02], [-7.0828e-03, -1.5942e-03, -2.5189e-02, 3.1235e-02, 1.0274e-02],
[-6.7937e-03, -6.4489e-03, -1.9903e-03, -2.9061e-03, -2.7987e-02], [-6.7937e-03, -6.4489e-03, -1.9903e-03, -2.9061e-03, -2.7987e-02],
[ 1.3823e-02, -2.3655e-02, -1.3454e-02, -1.0748e-02, -2.5365e-02]], [ 1.3823e-02, -2.3655e-02, -1.3454e-02, -1.0748e-02, -2.5365e-02]],
[[-1.7251e-02, 2.2574e-02, -3.1775e-02, 1.1056e-02, 3.0264e-02], [[-1.7251e-02, 2.2574e-02, -3.1775e-02, 1.1056e-02, 3.0264e-02],
[-2.4015e-03, 3.6797e-04, 3.1375e-02, -2.4610e-02, -1.7509e-02], [-2.4015e-03, 3.6797e-04, 3.1375e-02, -2.4610e-02, -1.7509e-02],
[ 1.0531e-02, 1.4460e-02, 2.5519e-02, -2.7404e-02, -3.1486e-02], [ 1.0531e-02, 1.4460e-02, 2.5519e-02, -2.7404e-02, -3.1486e-02],
[-7.5893e-03, 7.7746e-03, -7.8301e-03, 1.8347e-02, -1.2583e-02], [-7.5893e-03, 7.7746e-03, -7.8301e-03, 1.8347e-02, -1.2583e-02],
[-9.9493e-03, 1.7974e-03, -8.8425e-03, -1.0114e-02, -8.3833e-03]], [-9.9493e-03, 1.7974e-03, -8.8425e-03, -1.0114e-02, -8.3833e-03]],
[[-2.7485e-02, -1.0523e-02, -2.7561e-02, -3.0887e-02, 1.6148e-02], [[-2.7485e-02, -1.0523e-02, -2.7561e-02, -3.0887e-02, 1.6148e-02],
[-2.1342e-02, -3.1566e-02, -2.8830e-02, 2.3036e-02, 2.0477e-02], [-2.1342e-02, -3.1566e-02, -2.8830e-02, 2.3036e-02, 2.0477e-02],
[ 2.5157e-02, 1.3644e-02, 2.7742e-02, 2.7517e-02, 1.9181e-02], [ 2.5157e-02, 1.3644e-02, 2.7742e-02, 2.7517e-02, 1.9181e-02],
[ 2.8086e-02, -3.4166e-02, -2.4337e-02, 2.3341e-02, -1.8879e-02], [ 2.8086e-02, -3.4166e-02, -2.4337e-02, 2.3341e-02, -1.8879e-02],
[ 1.1548e-02, 3.8810e-03, 2.1140e-02, -1.9494e-02, 1.9646e-02]]], [ 1.1548e-02, 3.8810e-03, 2.1140e-02, -1.9494e-02, 1.9646e-02]]],
[[[ 1.4197e-02, 2.2693e-02, 2.4212e-02, 2.3371e-03, 3.2226e-02], [[[ 1.4197e-02, 2.2693e-02, 2.4212e-02, 2.3371e-03, 3.2226e-02],
[-1.2037e-03, 1.7354e-03, 3.5187e-02, 1.1291e-02, -4.9615e-04], [-1.2037e-03, 1.7354e-03, 3.5187e-02, 1.1291e-02, -4.9615e-04],
[ 9.9413e-03, -3.5207e-04, 9.8214e-03, 1.9830e-02, 3.4511e-02], [ 9.9413e-03, -3.5207e-04, 9.8214e-03, 1.9830e-02, 3.4511e-02],
[-5.0710e-03, 2.5797e-02, -2.4624e-02, -2.3307e-02, -1.6377e-02], [-5.0710e-03, 2.5797e-02, -2.4624e-02, -2.3307e-02, -1.6377e-02],
[-2.6477e-02, -5.8760e-04, -4.7716e-03, -2.0889e-02, 1.0995e-02]], [-2.6477e-02, -5.8760e-04, -4.7716e-03, -2.0889e-02, 1.0995e-02]],
[[-2.7908e-02, 3.3644e-02, 5.9671e-03, -1.6862e-02, -3.5059e-02], [[-2.7908e-02, 3.3644e-02, 5.9671e-03, -1.6862e-02, -3.5059e-02],
[ 1.6676e-02, 1.2755e-02, 1.5726e-02, -3.4257e-03, 2.4924e-02], [ 1.6676e-02, 1.2755e-02, 1.5726e-02, -3.4257e-03, 2.4924e-02],
[-2.3684e-02, 2.5474e-02, -1.5754e-02, -2.3486e-02, -2.9507e-03], [-2.3684e-02, 2.5474e-02, -1.5754e-02, -2.3486e-02, -2.9507e-03],
[ 1.2314e-02, 2.9314e-02, 3.3555e-02, 1.8871e-02, 1.8055e-02], [ 1.2314e-02, 2.9314e-02, 3.3555e-02, 1.8871e-02, 1.8055e-02],
[-1.4242e-02, 2.5115e-02, -2.2893e-02, -2.3078e-02, 1.8173e-02]], [-1.4242e-02, 2.5115e-02, -2.2893e-02, -2.3078e-02, 1.8173e-02]],
[[ 5.5846e-03, 2.1291e-03, 5.2495e-03, 1.2620e-03, -2.2444e-02], [[ 5.5846e-03, 2.1291e-03, 5.2495e-03, 1.2620e-03, -2.2444e-02],
[-1.6137e-02, 2.2707e-03, -2.3414e-02, -2.6367e-02, 1.8329e-02], [-1.6137e-02, 2.2707e-03, -2.3414e-02, -2.6367e-02, 1.8329e-02],
[-2.9248e-02, -4.8049e-03, -1.7246e-02, -1.4117e-02, -2.3186e-02], [-2.9248e-02, -4.8049e-03, -1.7246e-02, -1.4117e-02, -2.3186e-02],
[ 2.5687e-02, 3.1482e-02, -3.5057e-02, -2.6718e-02, -2.1352e-02], [ 2.5687e-02, 3.1482e-02, -3.5057e-02, -2.6718e-02, -2.1352e-02],
[ 6.3401e-03, 2.7100e-02, -8.5221e-04, -3.0839e-02, 3.2441e-02]], [ 6.3401e-03, 2.7100e-02, -8.5221e-04, -3.0839e-02, 3.2441e-02]],
..., ...,
[[-1.4497e-02, -1.4756e-02, 2.8516e-02, 6.1750e-03, -2.4559e-02], [[-1.4497e-02, -1.4756e-02, 2.8516e-02, 6.1750e-03, -2.4559e-02],
[-3.5086e-02, -2.1854e-02, -1.5571e-02, -1.5009e-02, 9.1230e-03], [-3.5086e-02, -2.1854e-02, -1.5571e-02, -1.5009e-02, 9.1230e-03],
[ 1.7028e-02, 8.5996e-03, 1.8000e-02, 9.9304e-03, 1.4644e-02], [ 1.7028e-02, 8.5996e-03, 1.8000e-02, 9.9304e-03, 1.4644e-02],
[ 2.5854e-02, -9.1312e-03, -1.5623e-02, 2.3103e-02, 1.4050e-02], [ 2.5854e-02, -9.1312e-03, -1.5623e-02, 2.3103e-02, 1.4050e-02],
[-2.2471e-02, 1.3548e-02, 1.4819e-02, -3.2382e-02, -3.3049e-02]], [-2.2471e-02, 1.3548e-02, 1.4819e-02, -3.2382e-02, -3.3049e-02]],
[[ 3.0104e-02, 4.6211e-03, -2.6502e-02, 8.1366e-04, -1.1328e-02], [[ 3.0104e-02, 4.6211e-03, -2.6502e-02, 8.1366e-04, -1.1328e-02],
[ 1.8561e-02, -1.8155e-02, 2.2092e-02, -1.9655e-02, 8.1872e-03], [ 1.8561e-02, -1.8155e-02, 2.2092e-02, -1.9655e-02, 8.1872e-03],
[ 1.3457e-02, -2.8087e-02, -2.1339e-02, 4.8732e-03, -3.0627e-02], [ 1.3457e-02, -2.8087e-02, -2.1339e-02, 4.8732e-03, -3.0627e-02],
[ 3.2684e-02, 3.3806e-02, -3.2461e-02, 1.3585e-02, -1.0189e-02], [ 3.2684e-02, 3.3806e-02, -3.2461e-02, 1.3585e-02, -1.0189e-02],
[ 3.5266e-02, 9.2128e-03, 1.3231e-02, -1.6795e-02, -7.2222e-03]], [ 3.5266e-02, 9.2128e-03, 1.3231e-02, -1.6795e-02, -7.2222e-03]],
[[-1.7422e-02, 2.3825e-02, -3.3179e-02, 2.6211e-02, -9.6366e-03], [[-1.7422e-02, 2.3825e-02, -3.3179e-02, 2.6211e-02, -9.6366e-03],
[-1.6651e-02, 2.9791e-02, -3.0735e-02, 1.5957e-02, -5.1611e-03], [-1.6651e-02, 2.9791e-02, -3.0735e-02, 1.5957e-02, -5.1611e-03],
[-2.5785e-02, 8.2464e-03, -2.6953e-02, -1.0548e-02, -3.5286e-02], [-2.5785e-02, 8.2464e-03, -2.6953e-02, -1.0548e-02, -3.5286e-02],
[-6.2504e-03, -1.3809e-02, -3.2342e-02, -5.4005e-03, 1.3821e-02], [-6.2504e-03, -1.3809e-02, -3.2342e-02, -5.4005e-03, 1.3821e-02],
[ 1.0036e-02, -2.3806e-02, -2.7316e-03, 8.0372e-03, -1.7473e-02]]], [ 1.0036e-02, -2.3806e-02, -2.7316e-03, 8.0372e-03, -1.7473e-02]]],
[[[-1.5131e-03, 2.1498e-02, 2.2542e-02, 7.8614e-04, 8.0929e-03], [[[-1.5131e-03, 2.1498e-02, 2.2542e-02, 7.8614e-04, 8.0929e-03],
[-2.2980e-02, -3.3852e-02, -2.1393e-02, -3.3051e-02, 1.5066e-02], [-2.2980e-02, -3.3852e-02, -2.1393e-02, -3.3051e-02, 1.5066e-02],
[ 1.2730e-02, 2.4478e-02, 2.8772e-02, 2.8062e-02, -3.9068e-03], [ 1.2730e-02, 2.4478e-02, 2.8772e-02, 2.8062e-02, -3.9068e-03],
[-9.3188e-03, -3.2090e-03, 1.1349e-03, -4.4513e-03, -2.2566e-02], [-9.3188e-03, -3.2090e-03, 1.1349e-03, -4.4513e-03, -2.2566e-02],
[ 1.5475e-02, -2.4088e-02, 4.7109e-03, -1.9576e-02, 2.5590e-03]], [ 1.5475e-02, -2.4088e-02, 4.7109e-03, -1.9576e-02, 2.5590e-03]],
[[ 3.4533e-02, 1.5805e-02, -8.1290e-03, -2.9473e-02, -2.7830e-02], [[ 3.4533e-02, 1.5805e-02, -8.1290e-03, -2.9473e-02, -2.7830e-02],
[-1.1189e-03, -2.3883e-02, -2.2408e-02, 8.0706e-03, -2.0912e-02], [-1.1189e-03, -2.3883e-02, -2.2408e-02, 8.0706e-03, -2.0912e-02],
[-6.0190e-03, 1.8530e-02, 3.1111e-02, 8.7140e-03, 1.4883e-02], [-6.0190e-03, 1.8530e-02, 3.1111e-02, 8.7140e-03, 1.4883e-02],
[ 1.0062e-02, -1.3259e-02, 7.0008e-03, -3.3429e-02, -2.5512e-02], [ 1.0062e-02, -1.3259e-02, 7.0008e-03, -3.3429e-02, -2.5512e-02],
[-3.3497e-02, 1.3196e-02, -2.5707e-02, 3.1772e-02, 3.0991e-03]], [-3.3497e-02, 1.3196e-02, -2.5707e-02, 3.1772e-02, 3.0991e-03]],
[[-1.4290e-02, 1.0892e-03, 2.0707e-02, 8.9846e-03, 2.7689e-02], [[-1.4290e-02, 1.0892e-03, 2.0707e-02, 8.9846e-03, 2.7689e-02],
[ 3.4180e-02, -2.9902e-02, -1.7447e-02, -2.6233e-02, -1.4028e-02], [ 3.4180e-02, -2.9902e-02, -1.7447e-02, -2.6233e-02, -1.4028e-02],
[-1.2295e-02, -2.0546e-02, -9.8043e-03, -2.7490e-02, -3.0622e-02], [-1.2295e-02, -2.0546e-02, -9.8043e-03, -2.7490e-02, -3.0622e-02],
[ 4.3674e-04, 2.9490e-02, 3.5398e-03, -4.6837e-03, 2.4393e-02], [ 4.3674e-04, 2.9490e-02, 3.5398e-03, -4.6837e-03, 2.4393e-02],
[-1.4666e-03, 1.1251e-02, -2.6159e-02, -2.1956e-02, -2.7467e-02]], [-1.4666e-03, 1.1251e-02, -2.6159e-02, -2.1956e-02, -2.7467e-02]],
..., ...,
[[-2.3644e-02, 3.0802e-02, -1.8002e-02, -3.0707e-02, -2.2172e-02], [[-2.3644e-02, 3.0802e-02, -1.8002e-02, -3.0707e-02, -2.2172e-02],
[-3.1019e-02, -1.9739e-02, -9.3868e-03, 7.9575e-03, -2.2624e-02], [-3.1019e-02, -1.9739e-02, -9.3868e-03, 7.9575e-03, -2.2624e-02],
[ 3.4551e-02, 2.0078e-02, -3.3876e-02, 3.1283e-02, 3.3183e-02], [ 3.4551e-02, 2.0078e-02, -3.3876e-02, 3.1283e-02, 3.3183e-02],
[-3.3918e-02, -2.3870e-02, -2.3575e-02, -2.9056e-02, -2.8837e-02], [-3.3918e-02, -2.3870e-02, -2.3575e-02, -2.9056e-02, -2.8837e-02],
[-2.5986e-02, -3.4845e-02, 7.3302e-03, -1.4608e-02, 1.0884e-02]], [-2.5986e-02, -3.4845e-02, 7.3302e-03, -1.4608e-02, 1.0884e-02]],
[[-1.4381e-04, -3.4048e-02, -2.3154e-02, -6.6671e-04, -1.3698e-03], [[-1.4381e-04, -3.4048e-02, -2.3154e-02, -6.6671e-04, -1.3698e-03],
[-2.2083e-02, -7.6509e-03, 1.5926e-02, 3.3870e-02, -8.7966e-03], [-2.2083e-02, -7.6509e-03, 1.5926e-02, 3.3870e-02, -8.7966e-03],
[ 1.0156e-02, -1.1124e-02, 3.2173e-02, -1.7401e-02, -1.3916e-02], [ 1.0156e-02, -1.1124e-02, 3.2173e-02, -1.7401e-02, -1.3916e-02],
[ 2.8669e-02, 7.6822e-03, -4.2535e-04, 2.8009e-02, 2.8256e-02], [ 2.8669e-02, 7.6822e-03, -4.2535e-04, 2.8009e-02, 2.8256e-02],
[-1.7353e-02, 2.9599e-02, 2.0556e-02, 2.6827e-02, -1.6913e-02]], [-1.7353e-02, 2.9599e-02, 2.0556e-02, 2.6827e-02, -1.6913e-02]],
[[-1.5121e-02, -2.5019e-02, 9.3863e-04, 1.8527e-02, 2.6718e-02], [[-1.5121e-02, -2.5019e-02, 9.3863e-04, 1.8527e-02, 2.6718e-02],
[-1.2542e-02, -9.8915e-03, -3.3949e-02, 2.0921e-02, 1.1817e-02], [-1.2542e-02, -9.8915e-03, -3.3949e-02, 2.0921e-02, 1.1817e-02],
[-3.3119e-02, 1.9118e-02, 2.8166e-02, 1.4008e-02, 7.0924e-03], [-3.3119e-02, 1.9118e-02, 2.8166e-02, 1.4008e-02, 7.0924e-03],
[-8.0425e-04, 4.4199e-03, 4.9465e-03, 6.0860e-03, -9.1247e-03], [-8.0425e-04, 4.4199e-03, 4.9465e-03, 6.0860e-03, -9.1247e-03],
[-8.3654e-03, 2.9056e-02, 2.2258e-02, -3.4614e-02, 2.7133e-02]]], [-8.3654e-03, 2.9056e-02, 2.2258e-02, -3.4614e-02, 2.7133e-02]]],
..., ...,
[[[ 2.4745e-02, 2.5447e-02, 7.8521e-03, 2.7326e-02, 4.1176e-03], [[[ 2.4745e-02, 2.5447e-02, 7.8521e-03, 2.7326e-02, 4.1176e-03],
[-9.2149e-03, -2.6898e-02, 1.9378e-02, -1.9195e-02, -3.2935e-02], [-9.2149e-03, -2.6898e-02, 1.9378e-02, -1.9195e-02, -3.2935e-02],
[-3.1293e-04, -5.0356e-03, -2.7392e-02, -2.9394e-02, 3.3510e-02], [-3.1293e-04, -5.0356e-03, -2.7392e-02, -2.9394e-02, 3.3510e-02],
[ 2.8223e-02, 3.3830e-02, 1.9529e-02, 2.8768e-02, -3.0616e-02], [ 2.8223e-02, 3.3830e-02, 1.9529e-02, 2.8768e-02, -3.0616e-02],
[ 9.7381e-03, 3.0267e-02, -2.2226e-02, -2.2444e-03, -3.5101e-02]], [ 9.7381e-03, 3.0267e-02, -2.2226e-02, -2.2444e-03, -3.5101e-02]],
[[-1.6917e-02, 1.1137e-02, 1.9355e-02, -1.4685e-02, 3.0447e-02], [[-1.6917e-02, 1.1137e-02, 1.9355e-02, -1.4685e-02, 3.0447e-02],
[-3.4133e-02, -1.8560e-02, 1.5245e-02, 9.2734e-03, 1.3745e-02], [-3.4133e-02, -1.8560e-02, 1.5245e-02, 9.2734e-03, 1.3745e-02],
[-2.7426e-02, 1.0003e-02, -2.2804e-02, 1.2485e-02, -1.3537e-02], [-2.7426e-02, 1.0003e-02, -2.2804e-02, 1.2485e-02, -1.3537e-02],
[-6.3152e-03, 3.4890e-02, 2.8487e-02, -2.7528e-02, 2.7217e-02], [-6.3152e-03, 3.4890e-02, 2.8487e-02, -2.7528e-02, 2.7217e-02],
[ 1.2584e-02, 3.0535e-02, 1.0904e-02, -2.9158e-02, 2.5303e-02]], [ 1.2584e-02, 3.0535e-02, 1.0904e-02, -2.9158e-02, 2.5303e-02]],
[[ 2.6163e-02, 2.8920e-02, 3.4083e-02, 1.2934e-02, -3.4496e-02], [[ 2.6163e-02, 2.8920e-02, 3.4083e-02, 1.2934e-02, -3.4496e-02],
[ 3.2103e-02, 7.7680e-03, 2.3343e-02, 2.5724e-02, -2.5661e-02], [ 3.2103e-02, 7.7680e-03, 2.3343e-02, 2.5724e-02, -2.5661e-02],
[-1.9408e-02, 1.8801e-02, 2.1603e-02, 3.1151e-02, 1.0190e-03], [-1.9408e-02, 1.8801e-02, 2.1603e-02, 3.1151e-02, 1.0190e-03],
[ 2.7702e-02, -2.0514e-02, -1.2836e-02, -2.3691e-02, 2.1812e-03], [ 2.7702e-02, -2.0514e-02, -1.2836e-02, -2.3691e-02, 2.1812e-03],
[-1.8351e-02, 2.5027e-02, -3.0688e-02, 4.3821e-03, 1.9501e-02]], [-1.8351e-02, 2.5027e-02, -3.0688e-02, 4.3821e-03, 1.9501e-02]],
..., ...,
[[-2.8270e-02, -2.3117e-02, -1.6419e-02, -3.4316e-02, -2.0557e-02], [[-2.8270e-02, -2.3117e-02, -1.6419e-02, -3.4316e-02, -2.0557e-02],
[-5.8688e-03, 2.0617e-03, 9.5386e-03, 1.3106e-02, 2.9971e-02], [-5.8688e-03, 2.0617e-03, 9.5386e-03, 1.3106e-02, 2.9971e-02],
[ 3.3426e-02, -2.6644e-02, 1.8090e-02, 7.3730e-03, -6.1319e-03], [ 3.3426e-02, -2.6644e-02, 1.8090e-02, 7.3730e-03, -6.1319e-03],
[-2.1524e-03, 6.3283e-03, 2.8402e-02, 1.5039e-02, 2.4081e-02], [-2.1524e-03, 6.3283e-03, 2.8402e-02, 1.5039e-02, 2.4081e-02],
[ 3.0103e-02, -1.2804e-02, -9.0004e-03, 1.6461e-02, -3.5154e-02]], [ 3.0103e-02, -1.2804e-02, -9.0004e-03, 1.6461e-02, -3.5154e-02]],
[[ 4.9777e-03, -8.0093e-03, -3.4668e-02, -8.8671e-03, -1.6355e-02], [[ 4.9777e-03, -8.0093e-03, -3.4668e-02, -8.8671e-03, -1.6355e-02],
[ 2.3159e-02, -3.2046e-02, -2.8806e-02, 3.3175e-02, -1.3727e-02], [ 2.3159e-02, -3.2046e-02, -2.8806e-02, 3.3175e-02, -1.3727e-02],
[ 2.8510e-02, -1.3080e-02, 3.2749e-02, 5.1974e-03, -1.6185e-02], [ 2.8510e-02, -1.3080e-02, 3.2749e-02, 5.1974e-03, -1.6185e-02],
[-3.3439e-02, -1.7652e-02, 3.2421e-02, 3.0137e-02, 7.4495e-03], [-3.3439e-02, -1.7652e-02, 3.2421e-02, 3.0137e-02, 7.4495e-03],
[-3.7330e-03, -7.3890e-03, -8.5357e-03, 2.2519e-02, -1.9516e-02]], [-3.7330e-03, -7.3890e-03, -8.5357e-03, 2.2519e-02, -1.9516e-02]],
[[ 6.3844e-03, -2.5492e-02, -1.6749e-02, -1.2231e-02, -1.0527e-02], [[ 6.3844e-03, -2.5492e-02, -1.6749e-02, -1.2231e-02, -1.0527e-02],
[-1.2054e-02, 3.7704e-03, 1.1407e-02, 1.4060e-02, -1.6531e-02], [-1.2054e-02, 3.7704e-03, 1.1407e-02, 1.4060e-02, -1.6531e-02],
[-1.1867e-02, 1.8624e-02, -1.1234e-02, -2.0407e-02, -1.2726e-02], [-1.1867e-02, 1.8624e-02, -1.1234e-02, -2.0407e-02, -1.2726e-02],
[-1.5739e-02, 1.7892e-02, -1.1010e-02, 1.8143e-02, 1.7330e-03], [-1.5739e-02, 1.7892e-02, -1.1010e-02, 1.8143e-02, 1.7330e-03],
[-2.6158e-04, -2.3484e-02, 2.8945e-02, -6.6862e-03, 3.0367e-02]]], [-2.6158e-04, -2.3484e-02, 2.8945e-02, -6.6862e-03, 3.0367e-02]]],
[[[-3.4974e-02, -2.3324e-02, 1.6922e-02, 2.8184e-02, -3.1625e-02], [[[-3.4974e-02, -2.3324e-02, 1.6922e-02, 2.8184e-02, -3.1625e-02],
[-8.9075e-03, -3.1058e-02, -6.1406e-03, 9.4089e-03, -1.1909e-02], [-8.9075e-03, -3.1058e-02, -6.1406e-03, 9.4089e-03, -1.1909e-02],
[-7.0569e-03, 2.3088e-02, 7.7149e-03, 2.7768e-02, 2.4761e-02], [-7.0569e-03, 2.3088e-02, 7.7149e-03, 2.7768e-02, 2.4761e-02],
[ 2.9902e-02, 7.4964e-03, -2.7370e-02, -1.8732e-02, -3.2662e-02], [ 2.9902e-02, 7.4964e-03, -2.7370e-02, -1.8732e-02, -3.2662e-02],
[-2.2868e-02, -2.0314e-02, -6.4008e-03, -6.3764e-03, 2.2569e-02]], [-2.2868e-02, -2.0314e-02, -6.4008e-03, -6.3764e-03, 2.2569e-02]],
[[-7.0775e-03, -4.1908e-03, 2.1538e-02, 1.4809e-02, -2.0912e-02], [[-7.0775e-03, -4.1908e-03, 2.1538e-02, 1.4809e-02, -2.0912e-02],
[ 2.1704e-02, 6.8497e-03, -3.4349e-02, -9.2698e-03, 4.3580e-03], [ 2.1704e-02, 6.8497e-03, -3.4349e-02, -9.2698e-03, 4.3580e-03],
[ 1.0022e-02, -2.7300e-03, -9.4724e-03, -7.5713e-04, 1.1330e-02], [ 1.0022e-02, -2.7300e-03, -9.4724e-03, -7.5713e-04, 1.1330e-02],
[-3.1236e-02, 3.3219e-02, 2.8622e-03, -2.5236e-02, 2.8182e-02], [-3.1236e-02, 3.3219e-02, 2.8622e-03, -2.5236e-02, 2.8182e-02],
[-1.1205e-02, 2.5173e-02, 1.3256e-02, -2.3062e-02, 1.1284e-02]], [-1.1205e-02, 2.5173e-02, 1.3256e-02, -2.3062e-02, 1.1284e-02]],
[[-9.4235e-03, -4.8488e-03, -2.8878e-02, 9.0096e-04, -1.7386e-02], [[-9.4235e-03, -4.8488e-03, -2.8878e-02, 9.0096e-04, -1.7386e-02],
[ 2.0669e-02, -2.1336e-02, 8.0332e-03, 1.7914e-02, -1.0405e-02], [ 2.0669e-02, -2.1336e-02, 8.0332e-03, 1.7914e-02, -1.0405e-02],
[-1.1635e-02, 3.3581e-02, -3.2376e-02, 2.9582e-02, 2.3664e-02], [-1.1635e-02, 3.3581e-02, -3.2376e-02, 2.9582e-02, 2.3664e-02],
[ 3.1593e-02, 3.4160e-02, -2.6020e-02, 8.8575e-03, 3.0042e-02], [ 3.1593e-02, 3.4160e-02, -2.6020e-02, 8.8575e-03, 3.0042e-02],
[ 2.8393e-02, -2.4792e-02, 3.2831e-02, -4.0711e-04, -3.4198e-02]], [ 2.8393e-02, -2.4792e-02, 3.2831e-02, -4.0711e-04, -3.4198e-02]],
..., ...,
[[ 1.0640e-02, -3.4773e-03, 1.7848e-02, 2.3339e-02, 6.5579e-03], [[ 1.0640e-02, -3.4773e-03, 1.7848e-02, 2.3339e-02, 6.5579e-03],
[-3.3516e-02, -9.5260e-03, 3.2948e-02, -1.4124e-02, -1.5335e-02], [-3.3516e-02, -9.5260e-03, 3.2948e-02, -1.4124e-02, -1.5335e-02],
[-2.6022e-02, -2.4680e-02, -2.5792e-02, -3.4441e-02, 5.2088e-03], [-2.6022e-02, -2.4680e-02, -2.5792e-02, -3.4441e-02, 5.2088e-03],
[ 1.1213e-02, -3.2379e-03, 2.8332e-02, 3.4966e-02, 1.9965e-02], [ 1.1213e-02, -3.2379e-03, 2.8332e-02, 3.4966e-02, 1.9965e-02],
[-6.4067e-05, 1.3670e-02, 3.2867e-03, 8.6834e-03, -1.4342e-02]], [-6.4067e-05, 1.3670e-02, 3.2867e-03, 8.6834e-03, -1.4342e-02]],
[[ 1.2845e-02, 1.6605e-03, -1.9191e-02, 1.9487e-02, -1.7243e-02], [[ 1.2845e-02, 1.6605e-03, -1.9191e-02, 1.9487e-02, -1.7243e-02],
[ 5.8301e-03, 1.1036e-02, -2.3355e-02, 1.0064e-02, 1.4885e-03], [ 5.8301e-03, 1.1036e-02, -2.3355e-02, 1.0064e-02, 1.4885e-03],
[ 3.3737e-02, 2.7396e-02, 6.1884e-03, 2.8059e-03, 1.3686e-02], [ 3.3737e-02, 2.7396e-02, 6.1884e-03, 2.8059e-03, 1.3686e-02],
[-1.2152e-02, -5.8316e-04, -3.1750e-03, 8.5284e-03, 1.3943e-02], [-1.2152e-02, -5.8316e-04, -3.1750e-03, 8.5284e-03, 1.3943e-02],
[-1.9356e-02, -1.0950e-04, -3.4086e-02, -7.5568e-04, 1.3694e-03]], [-1.9356e-02, -1.0950e-04, -3.4086e-02, -7.5568e-04, 1.3694e-03]],
[[ 1.8200e-02, 2.1633e-02, -2.4332e-02, 2.3583e-02, 2.9063e-02], [[ 1.8200e-02, 2.1633e-02, -2.4332e-02, 2.3583e-02, 2.9063e-02],
[-1.3286e-02, 2.9338e-03, 6.6651e-03, 2.9277e-03, 1.3872e-02], [-1.3286e-02, 2.9338e-03, 6.6651e-03, 2.9277e-03, 1.3872e-02],
[-2.3295e-02, 2.3481e-02, 6.5986e-03, 2.2366e-02, -3.5175e-02], [-2.3295e-02, 2.3481e-02, 6.5986e-03, 2.2366e-02, -3.5175e-02],
[-4.7833e-03, -1.3990e-02, 1.0363e-02, -3.1975e-02, 4.7487e-03], [-4.7833e-03, -1.3990e-02, 1.0363e-02, -3.1975e-02, 4.7487e-03],
[ 3.5129e-02, -3.8194e-03, -1.7300e-02, -1.9970e-02, 2.8379e-02]]], [ 3.5129e-02, -3.8194e-03, -1.7300e-02, -1.9970e-02, 2.8379e-02]]],
[[[ 1.4019e-02, 4.9214e-03, 1.0111e-02, -3.1889e-02, -2.7347e-02], [[[ 1.4019e-02, 4.9214e-03, 1.0111e-02, -3.1889e-02, -2.7347e-02],
[-8.1394e-03, -1.3291e-02, 2.4524e-02, -2.3494e-02, -2.0170e-03], [-8.1394e-03, -1.3291e-02, 2.4524e-02, -2.3494e-02, -2.0170e-03],
[-2.7336e-02, -1.9178e-02, -1.8323e-02, -8.8052e-03, 1.3123e-02], [-2.7336e-02, -1.9178e-02, -1.8323e-02, -8.8052e-03, 1.3123e-02],
[-3.5069e-02, -2.1669e-02, -1.1426e-02, 3.0186e-02, 1.9609e-02], [-3.5069e-02, -2.1669e-02, -1.1426e-02, 3.0186e-02, 1.9609e-02],
[-2.8254e-02, 1.6397e-02, -1.5657e-02, -2.0507e-02, -2.3217e-02]], [-2.8254e-02, 1.6397e-02, -1.5657e-02, -2.0507e-02, -2.3217e-02]],
[[ 8.5709e-03, -2.3691e-02, 1.7627e-02, 3.5223e-02, 2.6195e-03], [[ 8.5709e-03, -2.3691e-02, 1.7627e-02, 3.5223e-02, 2.6195e-03],
[-3.0899e-02, -3.4378e-02, 6.3869e-03, 3.2724e-03, -3.1556e-02], [-3.0899e-02, -3.4378e-02, 6.3869e-03, 3.2724e-03, -3.1556e-02],
[ 2.6143e-02, -2.5520e-02, -1.4755e-02, 3.2573e-02, 2.3502e-02], [ 2.6143e-02, -2.5520e-02, -1.4755e-02, 3.2573e-02, 2.3502e-02],
[-1.6250e-02, -4.1612e-03, -1.8213e-02, 3.1232e-02, 2.3861e-02], [-1.6250e-02, -4.1612e-03, -1.8213e-02, 3.1232e-02, 2.3861e-02],
[-2.1876e-02, -1.5494e-02, 3.0345e-03, 1.4424e-02, 2.9874e-02]], [-2.1876e-02, -1.5494e-02, 3.0345e-03, 1.4424e-02, 2.9874e-02]],
[[-2.5268e-02, 2.2410e-02, 2.3594e-02, 2.0579e-02, 1.6479e-02], [[-2.5268e-02, 2.2410e-02, 2.3594e-02, 2.0579e-02, 1.6479e-02],
[ 2.8202e-02, 3.0909e-03, -2.5594e-02, 2.6350e-02, 2.8560e-02], [ 2.8202e-02, 3.0909e-03, -2.5594e-02, 2.6350e-02, 2.8560e-02],
[-2.0345e-02, 3.2249e-02, 6.1006e-03, 2.1407e-02, -1.6071e-02], [-2.0345e-02, 3.2249e-02, 6.1006e-03, 2.1407e-02, -1.6071e-02],
[ 1.8856e-02, 2.9315e-02, -2.6081e-02, -1.4907e-02, -2.5855e-02], [ 1.8856e-02, 2.9315e-02, -2.6081e-02, -1.4907e-02, -2.5855e-02],
[ 4.3871e-03, -1.1070e-02, -4.8881e-03, 1.8845e-02, -1.4769e-03]], [ 4.3871e-03, -1.1070e-02, -4.8881e-03, 1.8845e-02, -1.4769e-03]],
..., ...,
[[ 1.2379e-02, 7.2388e-03, -1.4122e-02, 1.8673e-02, 1.8472e-02], [[ 1.2379e-02, 7.2388e-03, -1.4122e-02, 1.8673e-02, 1.8472e-02],
[-3.0360e-02, -2.3880e-02, -3.0080e-03, 1.0441e-02, 4.9348e-03], [-3.0360e-02, -2.3880e-02, -3.0080e-03, 1.0441e-02, 4.9348e-03],
[-3.0038e-02, 2.0413e-02, 2.8522e-02, -2.0159e-02, -1.8271e-02], [-3.0038e-02, 2.0413e-02, 2.8522e-02, -2.0159e-02, -1.8271e-02],
[-1.3495e-02, 9.2360e-03, -2.6240e-02, -2.5488e-02, 2.7568e-02], [-1.3495e-02, 9.2360e-03, -2.6240e-02, -2.5488e-02, 2.7568e-02],
[-1.5597e-02, -2.8057e-02, -2.9999e-02, -1.4414e-02, -4.7575e-03]], [-1.5597e-02, -2.8057e-02, -2.9999e-02, -1.4414e-02, -4.7575e-03]],
[[ 1.8291e-02, -1.6408e-02, -1.8680e-02, -2.2817e-02, -3.1854e-02], [[ 1.8291e-02, -1.6408e-02, -1.8680e-02, -2.2817e-02, -3.1854e-02],
[-2.9158e-02, 3.5350e-02, -6.7355e-04, 7.1955e-03, -2.8063e-02], [-2.9158e-02, 3.5350e-02, -6.7355e-04, 7.1955e-03, -2.8063e-02],
[ 1.7034e-02, 7.2418e-04, -1.2859e-02, 2.6168e-02, 1.9541e-02], [ 1.7034e-02, 7.2418e-04, -1.2859e-02, 2.6168e-02, 1.9541e-02],
[-3.5163e-03, 1.3653e-02, -2.5039e-02, 1.7310e-02, 1.2980e-02], [-3.5163e-03, 1.3653e-02, -2.5039e-02, 1.7310e-02, 1.2980e-02],
[-3.1598e-02, 1.0932e-02, -1.6623e-02, 2.7386e-03, -1.7496e-02]], [-3.1598e-02, 1.0932e-02, -1.6623e-02, 2.7386e-03, -1.7496e-02]],
[[ 3.2967e-03, 9.0435e-03, -6.3722e-03, 6.3557e-06, -1.6582e-04], [[ 3.2967e-03, 9.0435e-03, -6.3722e-03, 6.3557e-06, -1.6582e-04],
[-1.2603e-02, 2.8532e-02, 2.7132e-02, -3.3654e-02, -2.3288e-02], [-1.2603e-02, 2.8532e-02, 2.7132e-02, -3.3654e-02, -2.3288e-02],
[-3.3183e-02, -2.8453e-02, -3.4745e-02, 1.1531e-03, 1.3664e-02], [-3.3183e-02, -2.8453e-02, -3.4745e-02, 1.1531e-03, 1.3664e-02],
[ 3.2060e-02, 2.5101e-02, 2.1017e-02, 5.9951e-03, 2.6360e-03], [ 3.2060e-02, 2.5101e-02, 2.1017e-02, 5.9951e-03, 2.6360e-03],
[-2.0098e-02, 9.6565e-03, 1.4130e-02, -2.1930e-02, -1.9950e-03]]]]), tensor([ 0.0258, -0.0098, 0.0090, -0.0266, -0.0003, 0.0192, -0.0012, -0.0049, [-2.0098e-02, 9.6565e-03, 1.4130e-02, -2.1930e-02, -1.9950e-03]]]]), tensor([ 0.0258, -0.0098, 0.0090, -0.0266, -0.0003, 0.0192, -0.0012, -0.0049,
0.0099, 0.0069, 0.0019, 0.0319, 0.0208, 0.0209, -0.0220, -0.0238, 0.0099, 0.0069, 0.0019, 0.0319, 0.0208, 0.0209, -0.0220, -0.0238,
0.0177, -0.0307, 0.0308, 0.0066, -0.0072, -0.0034, 0.0152, 0.0293, 0.0177, -0.0307, 0.0308, 0.0066, -0.0072, -0.0034, 0.0152, 0.0293,
0.0169, 0.0327, -0.0059, 0.0043, 0.0057, 0.0174, -0.0194, -0.0216, 0.0169, 0.0327, -0.0059, 0.0043, 0.0057, 0.0174, -0.0194, -0.0216,
-0.0226, -0.0270, -0.0041, 0.0242, -0.0233, 0.0118, 0.0283, -0.0090, -0.0226, -0.0270, -0.0041, 0.0242, -0.0233, 0.0118, 0.0283, -0.0090,
-0.0180, -0.0280, -0.0086, -0.0122, 0.0003, -0.0277, -0.0038, 0.0353, -0.0180, -0.0280, -0.0086, -0.0122, 0.0003, -0.0277, -0.0038, 0.0353,
-0.0082, -0.0109, -0.0251, 0.0068, 0.0323, 0.0098, -0.0314, 0.0030, -0.0082, -0.0109, -0.0251, 0.0068, 0.0323, 0.0098, -0.0314, 0.0030,
0.0113, 0.0020, -0.0341, 0.0107, 0.0091, 0.0199, -0.0185, 0.0127]), tensor([[-0.0081, -0.0059, -0.0143, ..., 0.0117, -0.0148, 0.0108], 0.0113, 0.0020, -0.0341, 0.0107, 0.0091, 0.0199, -0.0185, 0.0127]), tensor([[-0.0081, -0.0059, -0.0143, ..., 0.0117, -0.0148, 0.0108],
[ 0.0151, 0.0105, 0.0061, ..., 0.0053, -0.0170, 0.0006], [ 0.0151, 0.0105, 0.0061, ..., 0.0053, -0.0170, 0.0006],
[-0.0148, -0.0145, -0.0035, ..., 0.0020, -0.0044, -0.0102], [-0.0148, -0.0145, -0.0035, ..., 0.0020, -0.0044, -0.0102],
..., ...,
[-0.0045, 0.0173, -0.0167, ..., -0.0103, 0.0056, -0.0051], [-0.0045, 0.0173, -0.0167, ..., -0.0103, 0.0056, -0.0051],
[ 0.0080, 0.0030, -0.0087, ..., 0.0165, -0.0167, 0.0033], [ 0.0080, 0.0030, -0.0087, ..., 0.0165, -0.0167, 0.0033],
[ 0.0098, 0.0124, 0.0158, ..., -0.0083, -0.0151, 0.0059]]), tensor([ 0.0140, -0.0175, 0.0051, -0.0030, -0.0019, -0.0134, -0.0133, -0.0009, [ 0.0098, 0.0124, 0.0158, ..., -0.0083, -0.0151, 0.0059]]), tensor([ 0.0140, -0.0175, 0.0051, -0.0030, -0.0019, -0.0134, -0.0133, -0.0009,
0.0023, 0.0163, -0.0042, -0.0130, -0.0113, -0.0176, -0.0152, 0.0139, 0.0023, 0.0163, -0.0042, -0.0130, -0.0113, -0.0176, -0.0152, 0.0139,
0.0032, 0.0127, 0.0173, 0.0080, -0.0009, -0.0097, -0.0020, 0.0114, 0.0032, 0.0127, 0.0173, 0.0080, -0.0009, -0.0097, -0.0020, 0.0114,
0.0059, -0.0070, 0.0143, -0.0143, 0.0008, 0.0067, -0.0162, 0.0094, 0.0059, -0.0070, 0.0143, -0.0143, 0.0008, 0.0067, -0.0162, 0.0094,
0.0165, 0.0118, -0.0017, -0.0064, -0.0134, -0.0012, 0.0107, 0.0039, 0.0165, 0.0118, -0.0017, -0.0064, -0.0134, -0.0012, 0.0107, 0.0039,
-0.0173, -0.0093, -0.0028, -0.0070, 0.0075, 0.0142, 0.0102, 0.0140, -0.0173, -0.0093, -0.0028, -0.0070, 0.0075, 0.0142, 0.0102, 0.0140,
-0.0120, 0.0057, 0.0095, 0.0077, -0.0151, -0.0022, 0.0130, 0.0005, -0.0120, 0.0057, 0.0095, 0.0077, -0.0151, -0.0022, 0.0130, 0.0005,
0.0123, 0.0055, 0.0120, -0.0058, -0.0142, -0.0129, 0.0048, -0.0020, 0.0123, 0.0055, 0.0120, -0.0058, -0.0142, -0.0129, 0.0048, -0.0020,
0.0016, 0.0016, 0.0061, -0.0174, -0.0123, -0.0010, 0.0032, -0.0170, 0.0016, 0.0016, 0.0061, -0.0174, -0.0123, -0.0010, 0.0032, -0.0170,
0.0020, 0.0172, 0.0146, -0.0086, 0.0032, 0.0093, 0.0139, 0.0085, 0.0020, 0.0172, 0.0146, -0.0086, 0.0032, 0.0093, 0.0139, 0.0085,
0.0061, -0.0039, 0.0154, -0.0014, -0.0082, -0.0030, 0.0009, 0.0150, 0.0061, -0.0039, 0.0154, -0.0014, -0.0082, -0.0030, 0.0009, 0.0150,
0.0054, 0.0015, 0.0047, 0.0063, 0.0095, 0.0081, 0.0125, -0.0027, 0.0054, 0.0015, 0.0047, 0.0063, 0.0095, 0.0081, 0.0125, -0.0027,
0.0130, -0.0039, -0.0038, 0.0035, 0.0163, -0.0174, 0.0106, 0.0007, 0.0130, -0.0039, -0.0038, 0.0035, 0.0163, -0.0174, 0.0106, 0.0007,
0.0087, 0.0070, -0.0141, -0.0139, -0.0068, -0.0126, 0.0130, -0.0088, 0.0087, 0.0070, -0.0141, -0.0139, -0.0068, -0.0126, 0.0130, -0.0088,
0.0088, -0.0121, -0.0087, 0.0134, 0.0101, -0.0139, -0.0002, -0.0055, 0.0088, -0.0121, -0.0087, 0.0134, 0.0101, -0.0139, -0.0002, -0.0055,
0.0039, -0.0103, -0.0068, 0.0082, 0.0059, -0.0069, -0.0138, -0.0164, 0.0039, -0.0103, -0.0068, 0.0082, 0.0059, -0.0069, -0.0138, -0.0164,
-0.0138, 0.0156, 0.0020, 0.0154, -0.0147, -0.0134, 0.0118, 0.0084, -0.0138, 0.0156, 0.0020, 0.0154, -0.0147, -0.0134, 0.0118, 0.0084,
0.0071, 0.0033, 0.0166, -0.0116, -0.0003, 0.0178, -0.0074, -0.0174, 0.0071, 0.0033, 0.0166, -0.0116, -0.0003, 0.0178, -0.0074, -0.0174,
-0.0094, 0.0002, 0.0010, -0.0066, -0.0090, -0.0143, 0.0048, -0.0169, -0.0094, 0.0002, 0.0010, -0.0066, -0.0090, -0.0143, 0.0048, -0.0169,
-0.0163, -0.0105, -0.0090, -0.0105, -0.0167, -0.0037, 0.0024, 0.0075, -0.0163, -0.0105, -0.0090, -0.0105, -0.0167, -0.0037, 0.0024, 0.0075,
-0.0023, -0.0143, 0.0052, -0.0041, -0.0139, -0.0123, 0.0161, -0.0053, -0.0023, -0.0143, 0.0052, -0.0041, -0.0139, -0.0123, 0.0161, -0.0053,
-0.0059, 0.0012, -0.0077, -0.0133, -0.0012, -0.0127, 0.0005, -0.0130, -0.0059, 0.0012, -0.0077, -0.0133, -0.0012, -0.0127, 0.0005, -0.0130,
-0.0176, -0.0114, 0.0139, -0.0153, 0.0124, 0.0151, -0.0092, -0.0168, -0.0176, -0.0114, 0.0139, -0.0153, 0.0124, 0.0151, -0.0092, -0.0168,
-0.0070, 0.0114, -0.0125, 0.0129, -0.0066, 0.0150, 0.0046, 0.0043, -0.0070, 0.0114, -0.0125, 0.0129, -0.0066, 0.0150, 0.0046, 0.0043,
0.0111, 0.0087, -0.0170, -0.0174, -0.0116, 0.0087, 0.0146, 0.0078, 0.0111, 0.0087, -0.0170, -0.0174, -0.0116, 0.0087, 0.0146, 0.0078,
-0.0055, -0.0119, -0.0041, -0.0017, -0.0093, 0.0170, 0.0093, -0.0026, -0.0055, -0.0119, -0.0041, -0.0017, -0.0093, 0.0170, 0.0093, -0.0026,
-0.0117, 0.0008, -0.0084, -0.0079, -0.0071, -0.0148, -0.0170, 0.0022, -0.0117, 0.0008, -0.0084, -0.0079, -0.0071, -0.0148, -0.0170, 0.0022,
-0.0049, 0.0149, 0.0078, -0.0042, 0.0118, -0.0100, 0.0136, 0.0112, -0.0049, 0.0149, 0.0078, -0.0042, 0.0118, -0.0100, 0.0136, 0.0112,
-0.0024, 0.0149, -0.0113, -0.0128, 0.0030, 0.0005, -0.0037, 0.0042, -0.0024, 0.0149, -0.0113, -0.0128, 0.0030, 0.0005, -0.0037, 0.0042,
0.0034, -0.0113, 0.0158, 0.0071, 0.0074, -0.0136, -0.0155, 0.0028, 0.0034, -0.0113, 0.0158, 0.0071, 0.0074, -0.0136, -0.0155, 0.0028,
0.0136, 0.0177, -0.0068, 0.0063, 0.0021, -0.0085, 0.0044, 0.0164, 0.0136, 0.0177, -0.0068, 0.0063, 0.0021, -0.0085, 0.0044, 0.0164,
-0.0075, 0.0109, 0.0103, -0.0105, -0.0167, 0.0114, 0.0093, -0.0098, -0.0075, 0.0109, 0.0103, -0.0105, -0.0167, 0.0114, 0.0093, -0.0098,
0.0115, -0.0011, 0.0097, 0.0087, 0.0075, -0.0120, 0.0011, -0.0067, 0.0115, -0.0011, 0.0097, 0.0087, 0.0075, -0.0120, 0.0011, -0.0067,
0.0058, -0.0050, -0.0077, 0.0039, -0.0112, -0.0015, -0.0152, 0.0074, 0.0058, -0.0050, -0.0077, 0.0039, -0.0112, -0.0015, -0.0152, 0.0074,
-0.0085, 0.0085, 0.0060, -0.0059, -0.0125, -0.0065, -0.0160, 0.0137, -0.0085, 0.0085, 0.0060, -0.0059, -0.0125, -0.0065, -0.0160, 0.0137,
0.0145, -0.0097, -0.0032, 0.0059, -0.0137, -0.0119, 0.0141, -0.0012, 0.0145, -0.0097, -0.0032, 0.0059, -0.0137, -0.0119, 0.0141, -0.0012,
0.0105, -0.0023, 0.0089, -0.0011, 0.0122, 0.0144, -0.0047, 0.0127, 0.0105, -0.0023, 0.0089, -0.0011, 0.0122, 0.0144, -0.0047, 0.0127,
-0.0046, 0.0057, -0.0112, 0.0062, 0.0176, 0.0162, 0.0004, -0.0061, -0.0046, 0.0057, -0.0112, 0.0062, 0.0176, 0.0162, 0.0004, -0.0061,
-0.0010, -0.0111, -0.0037, -0.0083, 0.0007, -0.0071, 0.0121, -0.0104, -0.0010, -0.0111, -0.0037, -0.0083, 0.0007, -0.0071, 0.0121, -0.0104,
0.0169, -0.0023, -0.0051, -0.0170, -0.0123, 0.0005, -0.0006, -0.0099, 0.0169, -0.0023, -0.0051, -0.0170, -0.0123, 0.0005, -0.0006, -0.0099,
0.0049, -0.0009, 0.0176, 0.0136, -0.0070, 0.0170, -0.0169, 0.0073, 0.0049, -0.0009, 0.0176, 0.0136, -0.0070, 0.0170, -0.0169, 0.0073,
-0.0058, -0.0111, -0.0014, -0.0076, -0.0080, 0.0006, 0.0075, 0.0064, -0.0058, -0.0111, -0.0014, -0.0076, -0.0080, 0.0006, 0.0075, 0.0064,
-0.0090, -0.0023, 0.0173, 0.0155, -0.0130, 0.0142, 0.0103, -0.0153, -0.0090, -0.0023, 0.0173, 0.0155, -0.0130, 0.0142, 0.0103, -0.0153,
-0.0011, 0.0139, 0.0086, -0.0130, 0.0029, 0.0074, -0.0003, 0.0030, -0.0011, 0.0139, 0.0086, -0.0130, 0.0029, 0.0074, -0.0003, 0.0030,
-0.0004, 0.0141, -0.0008, -0.0158, 0.0103, -0.0035, 0.0115, -0.0030, -0.0004, 0.0141, -0.0008, -0.0158, 0.0103, -0.0035, 0.0115, -0.0030,
-0.0049, -0.0023, -0.0098, 0.0058, -0.0002, 0.0170, 0.0124, 0.0086, -0.0049, -0.0023, -0.0098, 0.0058, -0.0002, 0.0170, 0.0124, 0.0086,
-0.0128, -0.0120, 0.0043, 0.0066, 0.0069, -0.0064, 0.0058, -0.0016, -0.0128, -0.0120, 0.0043, 0.0066, 0.0069, -0.0064, 0.0058, -0.0016,
0.0105, 0.0117, 0.0121, -0.0135, 0.0078, -0.0032, -0.0041, 0.0038, 0.0105, 0.0117, 0.0121, -0.0135, 0.0078, -0.0032, -0.0041, 0.0038,
0.0156, 0.0073, 0.0012, 0.0168, -0.0071, -0.0160, 0.0059, 0.0089, 0.0156, 0.0073, 0.0012, 0.0168, -0.0071, -0.0160, 0.0059, 0.0089,
-0.0043, 0.0019, -0.0178, 0.0018, 0.0017, -0.0090, 0.0136, -0.0167, -0.0043, 0.0019, -0.0178, 0.0018, 0.0017, -0.0090, 0.0136, -0.0167,
0.0018, 0.0094, -0.0153, -0.0061, -0.0172, -0.0072, -0.0088, -0.0058, 0.0018, 0.0094, -0.0153, -0.0061, -0.0172, -0.0072, -0.0088, -0.0058,
0.0103, -0.0004, 0.0163, 0.0048, 0.0093, 0.0106, -0.0093, 0.0023, 0.0103, -0.0004, 0.0163, 0.0048, 0.0093, 0.0106, -0.0093, 0.0023,
0.0003, -0.0150, -0.0174, -0.0041, -0.0008, -0.0068, 0.0174, 0.0111, 0.0003, -0.0150, -0.0174, -0.0041, -0.0008, -0.0068, 0.0174, 0.0111,
0.0168, 0.0034, -0.0021, 0.0051, 0.0053, 0.0017, -0.0137, 0.0046, 0.0168, 0.0034, -0.0021, 0.0051, 0.0053, 0.0017, -0.0137, 0.0046,
-0.0005, -0.0071, -0.0112, 0.0125, 0.0107, -0.0157, -0.0031, -0.0079, -0.0005, -0.0071, -0.0112, 0.0125, 0.0107, -0.0157, -0.0031, -0.0079,
-0.0030, 0.0099, -0.0033, 0.0061, 0.0009, 0.0099, -0.0055, -0.0099, -0.0030, 0.0099, -0.0033, 0.0061, 0.0009, 0.0099, -0.0055, -0.0099,
0.0057, 0.0126, -0.0092, -0.0071, -0.0156, -0.0092, -0.0086, 0.0043, 0.0057, 0.0126, -0.0092, -0.0071, -0.0156, -0.0092, -0.0086, 0.0043,
0.0026, 0.0127, -0.0045, 0.0163, -0.0036, 0.0123, 0.0060, 0.0123, 0.0026, 0.0127, -0.0045, 0.0163, -0.0036, 0.0123, 0.0060, 0.0123,
-0.0040, 0.0074, -0.0113, -0.0036, -0.0157, -0.0103, -0.0140, 0.0143, -0.0040, 0.0074, -0.0113, -0.0036, -0.0157, -0.0103, -0.0140, 0.0143,
0.0035, -0.0015, -0.0158, -0.0157, -0.0079, 0.0111, 0.0134, 0.0094, 0.0035, -0.0015, -0.0158, -0.0157, -0.0079, 0.0111, 0.0134, 0.0094,
-0.0120, -0.0178, -0.0065, -0.0007, 0.0016, -0.0102, -0.0133, -0.0095, -0.0120, -0.0178, -0.0065, -0.0007, 0.0016, -0.0102, -0.0133, -0.0095,
-0.0143, -0.0050, -0.0029, -0.0060, -0.0126, 0.0008, 0.0079, 0.0073, -0.0143, -0.0050, -0.0029, -0.0060, -0.0126, 0.0008, 0.0079, 0.0073,
-0.0082, -0.0120, 0.0162, -0.0058, 0.0126, 0.0147, -0.0131, -0.0136, -0.0082, -0.0120, 0.0162, -0.0058, 0.0126, 0.0147, -0.0131, -0.0136,
-0.0013, -0.0102, -0.0065, -0.0032, -0.0055, 0.0150, -0.0159, -0.0142]), tensor([[-0.0060, -0.0238, -0.0167, ..., -0.0295, -0.0283, 0.0013], -0.0013, -0.0102, -0.0065, -0.0032, -0.0055, 0.0150, -0.0159, -0.0142]), tensor([[-0.0060, -0.0238, -0.0167, ..., -0.0295, -0.0283, 0.0013],
[ 0.0272, -0.0164, 0.0261, ..., 0.0342, 0.0118, -0.0295], [ 0.0272, -0.0164, 0.0261, ..., 0.0342, 0.0118, -0.0295],
[-0.0431, -0.0081, -0.0005, ..., 0.0110, 0.0334, -0.0054], [-0.0431, -0.0081, -0.0005, ..., 0.0110, 0.0334, -0.0054],
..., ...,
[-0.0257, -0.0054, 0.0206, ..., -0.0191, 0.0412, -0.0227], [-0.0257, -0.0054, 0.0206, ..., -0.0191, 0.0412, -0.0227],
[ 0.0334, -0.0221, 0.0043, ..., 0.0383, -0.0305, 0.0005], [ 0.0334, -0.0221, 0.0043, ..., 0.0383, -0.0305, 0.0005],
[-0.0314, 0.0262, 0.0313, ..., -0.0048, 0.0006, 0.0426]]), tensor([-0.0431, -0.0134, -0.0107, 0.0229, 0.0287, -0.0393, -0.0060, -0.0009, [-0.0314, 0.0262, 0.0313, ..., -0.0048, 0.0006, 0.0426]]), tensor([-0.0431, -0.0134, -0.0107, 0.0229, 0.0287, -0.0393, -0.0060, -0.0009,
0.0412, -0.0344, -0.0020, 0.0278, -0.0317, -0.0283, 0.0094, 0.0142, 0.0412, -0.0344, -0.0020, 0.0278, -0.0317, -0.0283, 0.0094, 0.0142,
-0.0097, -0.0215, -0.0283, -0.0332, -0.0333, 0.0352, -0.0130, -0.0117, -0.0097, -0.0215, -0.0283, -0.0332, -0.0333, 0.0352, -0.0130, -0.0117,
-0.0346, -0.0030, -0.0087, -0.0178, 0.0225, 0.0430, 0.0153, -0.0233, -0.0346, -0.0030, -0.0087, -0.0178, 0.0225, 0.0430, 0.0153, -0.0233,
0.0231, -0.0374, -0.0200, 0.0007, 0.0341, 0.0077, -0.0098, -0.0249, 0.0231, -0.0374, -0.0200, 0.0007, 0.0341, 0.0077, -0.0098, -0.0249,
0.0400, -0.0381, -0.0155, 0.0188, -0.0362, 0.0095, -0.0313, 0.0109, 0.0400, -0.0381, -0.0155, 0.0188, -0.0362, 0.0095, -0.0313, 0.0109,
0.0298, -0.0086, 0.0114, -0.0137, -0.0284, 0.0243, 0.0336, 0.0356, 0.0298, -0.0086, 0.0114, -0.0137, -0.0284, 0.0243, 0.0336, 0.0356,
0.0076, 0.0008, -0.0358, -0.0435, 0.0303, 0.0354])] 0.0076, 0.0008, -0.0358, -0.0435, 0.0303, 0.0354])]
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
a = [(3, 2), (2, 5), (2, 6)] a = [(3, 2), (2, 5), (2, 6)]
#a.sort(reverse = True) #a.sort(reverse = True)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
b = [(13, 12), (12, 15), (12, 16)] b = [(13, 12), (12, 15), (12, 16)]
import torch import torch
a_t = torch.tensor(a) a_t = torch.tensor(a)
b_t = torch.tensor(b) b_t = torch.tensor(b)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
print(a_t) print(a_t)
print(b_t) print(b_t)
``` ```
%% Output %% Output
tensor([[3, 2], tensor([[3, 2],
[2, 5], [2, 5],
[2, 6]]) [2, 6]])
tensor([[13, 12], tensor([[13, 12],
[12, 15], [12, 15],
[12, 16]]) [12, 16]])
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
indices = torch.tensor([2, 3, 5]) indices = torch.tensor([2, 3, 5])
torch.flatten(a_t)[indices] = torch.flatten(b_t)[:3] torch.flatten(a_t)[indices] = torch.flatten(b_t)[:3]
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
c = torch.stack((torch.flatten(a_t), torch.flatten(b_t)), dim=-1) c = torch.stack((torch.flatten(a_t), torch.flatten(b_t)), dim=-1)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
print(a_t) print(a_t)
print(b_t) print(b_t)
``` ```
%% Output %% Output
tensor([[ 3, 2], tensor([[ 3, 2],
[13, 12], [13, 12],
[ 2, 12]]) [ 2, 12]])
tensor([[13, 12], tensor([[13, 12],
[12, 15], [12, 15],
[12, 16]]) [12, 16]])
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
c c
``` ```
%% Output %% Output
tensor([[ 3, 13], tensor([[ 3, 13],
[ 2, 12], [ 2, 12],
[ 2, 12], [ 2, 12],
[ 5, 15], [ 5, 15],
[ 2, 12], [ 2, 12],
[ 6, 16]]) [ 6, 16]])
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
c[-1] = 26 c[-1] = 26
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
c c
``` ```
%% Output %% Output
tensor([ 3, 2, 2, 5, 2, 6, 13, 12, 12, 15, 12, 26]) tensor([ 3, 2, 2, 5, 2, 6, 13, 12, 12, 15, 12, 26])
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
b b
``` ```
%% Output %% Output
[(13, 12), (12, 15), (12, 16)] [(13, 12), (12, 15), (12, 16)]
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
torch.tensor([1,2,3]).shape[0] torch.tensor([1,2,3]).shape[0]
``` ```
%% Output %% Output
3 3
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
from decentralizepy.datasets.Partitioner import DataPartitioner from decentralizepy.datasets.Partitioner import DataPartitioner
l = [1, 2, 3, 4, 5, 6, 7] l = [1, 2, 3, 4, 5, 6, 7]
e = len(l) // 3 e = len(l) // 3
frac = e / len(l) frac = e / len(l)
sizes = [frac] * 3 sizes = [frac] * 3
sizes[-1] += 1.0 - frac * 3 sizes[-1] += 1.0 - frac * 3
print("sizes: ", sizes) print("sizes: ", sizes)
for i in range(3): for i in range(3):
myPar = DataPartitioner(l, sizes).use(i) myPar = DataPartitioner(l, sizes).use(i)
for j in range(len(myPar)): for j in range(len(myPar)):
print(myPar.__getitem__(j), end=' ') print(myPar.__getitem__(j), end=' ')
print() print()
``` ```
%% Output %% Output
sizes: [0.2857142857142857, 0.2857142857142857, 0.4285714285714286] sizes: [0.2857142857142857, 0.2857142857142857, 0.4285714285714286]
2 3 2 3
6 5 6 5
1 4 7 1 4 7
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
w = 1 w = 1
p = 2 p = 2
i = w/p i = w/p
print(i) print(i)
``` ```
%% Output %% Output
0.5 0.5
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` ``` python
from torch import multiprocessing as mp
from decentralizepy.node.PeerSampler import PeerSampler
from decentralizepy.node.Node import Node
from decentralizepy.mappings.Linear import Linear
from decentralizepy.graphs.Regular import Regular
l = Linear(1, 6)
g = Regular(6, 2)
processes = [mp.Process(target = PeerSampler, args=[-1, 0, l, g, None]),
mp.Process(target = Node, args=[1, 0, l, g, None]),
mp.Process(target = Node, args=[2, 0, l, g, None]),
mp.Process(target = Node, args=[3, 0, l, g, None]),
mp.Process(target = Node, args=[4, 0, l, g, None]),
mp.Process(target = Node, args=[5, 0, l, g, None]),
mp.Process(target = Node, args=[6, 0, l, g, None]),
]
for p in processes:
p.start()
for p in processes:
p.join()
```
%% Cell type:code id: tags:
``` python
``` ```
......
eval/plot.py
View file @ d140bf44
...@@ -26,19 +26,27 @@ def get_stats(l): ...@@ -26,19 +26,27 @@ def get_stats(l):
return mean_dict, stdev_dict, min_dict, max_dict return mean_dict, stdev_dict, min_dict, max_dict
def plot(means, stdevs, mins, maxs, title, label, loc): def plot(means, stdevs, mins, maxs, title, label, loc, xlabel="communication rounds"):
plt.title(title) plt.title(title)
plt.xlabel("communication rounds") plt.xlabel(xlabel)
x_axis = list(means.keys()) x_axis = np.array(list(means.keys()))
y_axis = list(means.values()) y_axis = np.array(list(means.values()))
err = list(stdevs.values()) err = np.array(list(stdevs.values()))
plt.errorbar(x_axis, y_axis, yerr=err, label=label) plt.plot(x_axis, y_axis, label=label)
plt.fill_between(x_axis, y_axis - err, y_axis + err, alpha=0.4)
plt.legend(loc=loc) plt.legend(loc=loc)
def replace_dict_key(d_org: dict, d_other: dict):
result = {}
for x, y in d_org.items():
result[d_other[x]] = y
return result
def plot_results(path, centralized, data_machine="machine0", data_node=0): def plot_results(path, centralized, data_machine="machine0", data_node=0):
folders = os.listdir(path) folders = os.listdir(path)
if centralized.lower() in ['true', '1', 't', 'y', 'yes']: if centralized.lower() in ["true", "1", "t", "y", "yes"]:
centralized = True centralized = True
print("Centralized") print("Centralized")
else: else:
...@@ -66,23 +74,61 @@ def plot_results(path, centralized, data_machine="machine0", data_node=0): ...@@ -66,23 +74,61 @@ def plot_results(path, centralized, data_machine="machine0", data_node=0):
filepath = os.path.join(mf_path, f) filepath = os.path.join(mf_path, f)
with open(filepath, "r") as inf: with open(filepath, "r") as inf:
results.append(json.load(inf)) results.append(json.load(inf))
if folder.startswith("FL") or folder.startswith("Parameter Server"):
data_node = -1
else:
data_node = 0
with open(folder_path / data_machine / f"{data_node}_results.json", "r") as f: with open(folder_path / data_machine / f"{data_node}_results.json", "r") as f:
main_data = json.load(f) main_data = json.load(f)
main_data = [main_data] main_data = [main_data]
# Plotting bytes over time
plt.figure(10)
b_means, stdevs, mins, maxs = get_stats([x["total_bytes"] for x in results])
plot(b_means, stdevs, mins, maxs, "Total Bytes", folder, "lower right")
df = pd.DataFrame(
{
"mean": list(b_means.values()),
"std": list(stdevs.values()),
"nr_nodes": [len(results)] * len(b_means),
},
list(b_means.keys()),
columns=["mean", "std", "nr_nodes"],
)
df.to_csv(
os.path.join(path, "total_bytes_" + folder + ".csv"), index_label="rounds"
)
# Plot Training loss # Plot Training loss
plt.figure(1) plt.figure(1)
means, stdevs, mins, maxs = get_stats([x["train_loss"] for x in results]) means, stdevs, mins, maxs = get_stats([x["train_loss"] for x in results])
plot(means, stdevs, mins, maxs, "Training Loss", folder, "upper right") plot(means, stdevs, mins, maxs, "Training Loss", folder, "upper right")
correct_bytes = [b_means[x] for x in means]
df = pd.DataFrame( df = pd.DataFrame(
{ {
"mean": list(means.values()), "mean": list(means.values()),
"std": list(stdevs.values()), "std": list(stdevs.values()),
"nr_nodes": [len(results)] * len(means), "nr_nodes": [len(results)] * len(means),
"total_bytes": correct_bytes,
}, },
list(means.keys()), list(means.keys()),
columns=["mean", "std", "nr_nodes"], columns=["mean", "std", "nr_nodes", "total_bytes"],
) )
plt.figure(11)
means = replace_dict_key(means, b_means)
plot(
means,
stdevs,
mins,
maxs,
"Training Loss",
folder,
"upper right",
"Total Bytes per node",
)
df.to_csv( df.to_csv(
os.path.join(path, "train_loss_" + folder + ".csv"), index_label="rounds" os.path.join(path, "train_loss_" + folder + ".csv"), index_label="rounds"
) )
...@@ -98,10 +144,24 @@ def plot_results(path, centralized, data_machine="machine0", data_node=0): ...@@ -98,10 +144,24 @@ def plot_results(path, centralized, data_machine="machine0", data_node=0):
"mean": list(means.values()), "mean": list(means.values()),
"std": list(stdevs.values()), "std": list(stdevs.values()),
"nr_nodes": [len(results)] * len(means), "nr_nodes": [len(results)] * len(means),
"total_bytes": correct_bytes,
}, },
list(means.keys()), list(means.keys()),
columns=["mean", "std", "nr_nodes"], columns=["mean", "std", "nr_nodes", "total_bytes"],
)
plt.figure(12)
means = replace_dict_key(means, b_means)
plot(
means,
stdevs,
mins,
maxs,
"Testing Loss",
folder,
"upper right",
"Total Bytes per node",
) )
df.to_csv( df.to_csv(
os.path.join(path, "test_loss_" + folder + ".csv"), index_label="rounds" os.path.join(path, "test_loss_" + folder + ".csv"), index_label="rounds"
) )
...@@ -117,30 +177,27 @@ def plot_results(path, centralized, data_machine="machine0", data_node=0): ...@@ -117,30 +177,27 @@ def plot_results(path, centralized, data_machine="machine0", data_node=0):
"mean": list(means.values()), "mean": list(means.values()),
"std": list(stdevs.values()), "std": list(stdevs.values()),
"nr_nodes": [len(results)] * len(means), "nr_nodes": [len(results)] * len(means),
"total_bytes": correct_bytes,
}, },
list(means.keys()), list(means.keys()),
columns=["mean", "std", "nr_nodes"], columns=["mean", "std", "nr_nodes", "total_bytes"],
)
df.to_csv(
os.path.join(path, "test_acc_" + folder + ".csv"), index_label="rounds"
) )
plt.figure(6) plt.figure(13)
means, stdevs, mins, maxs = get_stats([x["grad_std"] for x in results]) means = replace_dict_key(means, b_means)
plot( plot(
means, means,
stdevs, stdevs,
mins, mins,
maxs, maxs,
"Gradient Variation over Nodes", "Testing Accuracy",
folder, folder,
"upper right", "lower right",
"Total Bytes per node",
) )
# Plot Testing loss df.to_csv(
plt.figure(7) os.path.join(path, "test_acc_" + folder + ".csv"), index_label="rounds"
means, stdevs, mins, maxs = get_stats([x["grad_mean"] for x in results])
plot(
means, stdevs, mins, maxs, "Gradient Magnitude Mean", folder, "upper right"
) )
# Collect total_bytes shared # Collect total_bytes shared
bytes_list = [] bytes_list = []
for x in results: for x in results:
...@@ -169,16 +226,21 @@ def plot_results(path, centralized, data_machine="machine0", data_node=0): ...@@ -169,16 +226,21 @@ def plot_results(path, centralized, data_machine="machine0", data_node=0):
data_means[folder] = list(means.values())[0] data_means[folder] = list(means.values())[0]
data_stdevs[folder] = list(stdevs.values())[0] data_stdevs[folder] = list(stdevs.values())[0]
plt.figure(10)
plt.savefig(os.path.join(path, "total_bytes.png"), dpi=300)
plt.figure(11)
plt.savefig(os.path.join(path, "bytes_train_loss.png"), dpi=300)
plt.figure(12)
plt.savefig(os.path.join(path, "bytes_test_loss.png"), dpi=300)
plt.figure(13)
plt.savefig(os.path.join(path, "bytes_test_acc.png"), dpi=300)
plt.figure(1) plt.figure(1)
plt.savefig(os.path.join(path, "train_loss.png"), dpi=300) plt.savefig(os.path.join(path, "train_loss.png"), dpi=300)
plt.figure(2) plt.figure(2)
plt.savefig(os.path.join(path, "test_loss.png"), dpi=300) plt.savefig(os.path.join(path, "test_loss.png"), dpi=300)
plt.figure(3) plt.figure(3)
plt.savefig(os.path.join(path, "test_acc.png"), dpi=300) plt.savefig(os.path.join(path, "test_acc.png"), dpi=300)
plt.figure(6)
plt.savefig(os.path.join(path, "grad_std.png"), dpi=300)
plt.figure(7)
plt.savefig(os.path.join(path, "grad_mean.png"), dpi=300)
# Plot total_bytes # Plot total_bytes
plt.figure(4) plt.figure(4)
plt.title("Data Shared") plt.title("Data Shared")
...@@ -257,5 +319,6 @@ if __name__ == "__main__": ...@@ -257,5 +319,6 @@ if __name__ == "__main__":
# The args are: # The args are:
# 1: the folder with the data # 1: the folder with the data
# 2: True/False: If True then the evaluation on the test set was centralized # 2: True/False: If True then the evaluation on the test set was centralized
# for federated learning folder name must start with "FL"!
plot_results(sys.argv[1], sys.argv[2]) plot_results(sys.argv[1], sys.argv[2])
# plot_parameters(sys.argv[1]) # plot_parameters(sys.argv[1])
eval/plotting_from_csv.py
View file @ d140bf44
...@@ -23,7 +23,7 @@ def plot(x_axis, means, stdevs, pos, nb_plots, title, label, loc, xlabel): ...@@ -23,7 +23,7 @@ def plot(x_axis, means, stdevs, pos, nb_plots, title, label, loc, xlabel):
def plot_results(path, epochs, global_epochs="True"): def plot_results(path, epochs, global_epochs="True"):
if global_epochs.lower() in ['true', '1', 't', 'y', 'yes']: if global_epochs.lower() in ["true", "1", "t", "y", "yes"]:
global_epochs = True global_epochs = True
else: else:
global_epochs = False global_epochs = False
...@@ -52,10 +52,12 @@ def plot_results(path, epochs, global_epochs="True"): ...@@ -52,10 +52,12 @@ def plot_results(path, epochs, global_epochs="True"):
if global_epochs: if global_epochs:
rounds = results_csv["rounds"].iloc[0] rounds = results_csv["rounds"].iloc[0]
print("Rounds: ", rounds) print("Rounds: ", rounds)
results_cr = results_csv[results_csv.rounds <= epochs*rounds] results_cr = results_csv[results_csv.rounds <= epochs * rounds]
means = results_cr["mean"].to_numpy() means = results_cr["mean"].to_numpy()
stdevs = results_cr["std"].to_numpy() stdevs = results_cr["std"].to_numpy()
x_axis = results_cr["rounds"].to_numpy() / rounds # list(np.arange(0, len(means), 1)) x_axis = (
results_cr["rounds"].to_numpy() / rounds
) # list(np.arange(0, len(means), 1))
x_label = "global epochs" x_label = "global epochs"
else: else:
results_cr = results_csv[results_csv.rounds <= epochs] results_cr = results_csv[results_csv.rounds <= epochs]
...@@ -85,10 +87,12 @@ def plot_results(path, epochs, global_epochs="True"): ...@@ -85,10 +87,12 @@ def plot_results(path, epochs, global_epochs="True"):
if global_epochs: if global_epochs:
rounds = results_csv["rounds"].iloc[0] rounds = results_csv["rounds"].iloc[0]
print("Rounds: ", rounds) print("Rounds: ", rounds)
results_cr = results_csv[results_csv.rounds <= epochs*rounds] results_cr = results_csv[results_csv.rounds <= epochs * rounds]
means = results_cr["mean"].to_numpy() means = results_cr["mean"].to_numpy()
stdevs = results_cr["std"].to_numpy() stdevs = results_cr["std"].to_numpy()
x_axis = results_cr["rounds"].to_numpy() / rounds # list(np.arange(0, len(means), 1)) x_axis = (
results_cr["rounds"].to_numpy() / rounds
) # list(np.arange(0, len(means), 1))
x_label = "global epochs" x_label = "global epochs"
else: else:
results_cr = results_csv[results_csv.rounds <= epochs] results_cr = results_csv[results_csv.rounds <= epochs]
...@@ -120,10 +124,12 @@ def plot_results(path, epochs, global_epochs="True"): ...@@ -120,10 +124,12 @@ def plot_results(path, epochs, global_epochs="True"):
if global_epochs: if global_epochs:
rounds = results_csv["rounds"].iloc[0] rounds = results_csv["rounds"].iloc[0]
print("Rounds: ", rounds) print("Rounds: ", rounds)
results_cr = results_csv[results_csv.rounds <= epochs*rounds] results_cr = results_csv[results_csv.rounds <= epochs * rounds]
means = results_cr["mean"].to_numpy() means = results_cr["mean"].to_numpy()
stdevs = results_cr["std"].to_numpy() stdevs = results_cr["std"].to_numpy()
x_axis = results_cr["rounds"].to_numpy() / rounds # list(np.arange(0, len(means), 1)) x_axis = (
results_cr["rounds"].to_numpy() / rounds
) # list(np.arange(0, len(means), 1))
x_label = "global epochs" x_label = "global epochs"
else: else:
results_cr = results_csv[results_csv.rounds <= epochs] results_cr = results_csv[results_csv.rounds <= epochs]
......
eval/run.sh
View file @ d140bf44
#!/bin/bash #!/bin/bash
script_path=$(realpath $(dirname $0))
decpy_path=/mnt/nfs/risharma/Gitlab/decentralizepy/eval # Working directory, where config files are read from and logs are written.
decpy_path=/mnt/nfs/$(whoami)/decpy_workingdir
cd $decpy_path cd $decpy_path
env_python=~/miniconda3/envs/decpy/bin/python3 # Python interpreter
graph=/mnt/nfs/risharma/Gitlab/tutorial/regular_16.txt env_python=python3
original_config=/mnt/nfs/risharma/Gitlab/tutorial/config_celeba_sharing.ini
config_file=~/tmp/config.ini # File regular_16.txt is available in /tutorial
graph=$decpy_path/regular_16.txt
# File config_celeba_sharing.ini is available in /tutorial
# In this config file, change addresses_filepath to correspond to your list of machines (example in /tutorial/ip.json)
original_config=$decpy_path/config_celeba_sharing.ini
# Local config file
config_file=/tmp/$(basename $original_config)
# Python script to be executed
eval_file=$script_path/testingPeerSampler.py
# General parameters
procs_per_machine=8 procs_per_machine=8
machines=2 machines=2
iterations=5 iterations=5
test_after=2 test_after=2
eval_file=testing.py
log_level=INFO log_level=INFO
m=`cat $(grep addresses_filepath $original_config | awk '{print $3}') | grep $(/sbin/ifconfig ens785 | grep 'inet ' | awk '{print $2}') | cut -d'"' -f2` m=`cat $(grep addresses_filepath $original_config | awk '{print $3}') | grep $(/sbin/ifconfig ens785 | grep 'inet ' | awk '{print $2}') | cut -d'"' -f2`
...@@ -19,6 +33,8 @@ echo M is $m ...@@ -19,6 +33,8 @@ echo M is $m
log_dir=$(date '+%Y-%m-%dT%H:%M')/machine$m log_dir=$(date '+%Y-%m-%dT%H:%M')/machine$m
mkdir -p $log_dir mkdir -p $log_dir
# Copy and manipulate the local config file
cp $original_config $config_file cp $original_config $config_file
# echo "alpha = 0.10" >> $config_file # echo "alpha = 0.10" >> $config_file
$env_python $eval_file -ro 0 -tea $test_after -ld $log_dir -mid $m -ps $procs_per_machine -ms $machines -is $iterations -gf $graph -ta $test_after -cf $config_file -ll $log_level -ctr 0 -cte 0 -wsd $log_dir
\ No newline at end of file $env_python $eval_file -ro 0 -tea $test_after -ld $log_dir -mid $m -ps $procs_per_machine -ms $machines -is $iterations -gf $graph -ta $test_after -cf $config_file -ll $log_level -wsd $log_dir
eval/run_xtimes.sh
View file @ d140bf44
#!/bin/bash #!/bin/bash
# Documentation # Documentation
# Note: documentation was not written for this run file, so actual behaviour may differ
# This bash file takes three inputs. The first argument (nfs_home) is the path to the nfs home directory. # This bash file takes three inputs. The first argument (nfs_home) is the path to the nfs home directory.
# The second one (python_bin) is the path to the python bin folder. # The second one (python_bin) is the path to the python bin folder.
# The last argument (logs_subfolder) is the path to the logs folder with respect to the nfs home directory. # The last argument (logs_subfolder) is the path to the logs folder with respect to the nfs home directory.
...@@ -18,8 +19,10 @@ ...@@ -18,8 +19,10 @@
# Each node needs a folder called 'tmp' in the user's home directory # Each node needs a folder called 'tmp' in the user's home directory
# #
# Note: # Note:
# - The script does not change the optimizer. All configs are writen to use SGD. # - The script does not change the optimizer. All configs are writen to use Adam.
# - The script will set '--test_after' and '--train_evaluate_after' such that it happens at the end of a global epoch. # For SGD these need to be changed manually
# - The script will set '--test_after' and '--train_evaluate_after' to comm_rounds_per_global_epoch, i.e., the eavaluation
# on the train set and on the test set is carried out every global epoch.
# - The '--reset_optimizer' option is set to 0, i.e., the optimizer is not reset after a communication round (only # - The '--reset_optimizer' option is set to 0, i.e., the optimizer is not reset after a communication round (only
# relevant for Adams and other optimizers with internal state) # relevant for Adams and other optimizers with internal state)
# #
...@@ -37,41 +40,40 @@ decpy_path=$nfs_home/decentralizepy/eval ...@@ -37,41 +40,40 @@ decpy_path=$nfs_home/decentralizepy/eval
cd $decpy_path cd $decpy_path
env_python=$python_bin/python3 env_python=$python_bin/python3
graph=96_regular.edges
config_file=~/tmp/config.ini config_file=~/tmp/config.ini
procs_per_machine=16 procs_per_machine=16
machines=6 machines=6
global_epochs=150 global_epochs=150
eval_file=testing.py eval_file=testingPeerSampler.py
log_level=INFO log_level=INFO
ip_machines=$nfs_home/$logs_subfolder/ip_addr_6Machines.json
ip_machines=$nfs_home/configs/ip_addr_6Machines.json
m=`cat $ip_machines | grep $(/sbin/ifconfig ens785 | grep 'inet ' | awk '{print $2}') | cut -d'"' -f2` m=`cat $ip_machines | grep $(/sbin/ifconfig ens785 | grep 'inet ' | awk '{print $2}') | cut -d'"' -f2`
export PYTHONFAULTHANDLER=1 export PYTHONFAULTHANDLER=1
# Base configs for which the gird search is done # Base configs for which the gird search is done
tests=("step_configs/config_femnist_partialmodel.ini" "step_configs/config_femnist_topkacc.ini" "step_configs/config_femnist_wavelet.ini") tests="$nfs_home/$logs_subfolder/config.ini"
#tests=("$nfs_home/$logs_subfolder/config_cifar_sharing.ini" "$nfs_home/$logs_subfolder/config_cifar_partialmodel.ini" "$nfs_home/$logs_subfolder/config_cifar_topkacc.ini" "$nfs_home/$logs_subfolder/config_cifar_topkaccRandomAlpha.ini" "$nfs_home/$logs_subfolder/config_cifar_subsampling.ini" "$nfs_home/$logs_subfolder/config_cifar_wavelet.ini" "$nfs_home/$logs_subfolder/config_cifar_waveletRandomAlpha.ini")
#tests=("$nfs_home/$logs_subfolder/config_cifar_partialmodel.ini" "$nfs_home/$logs_subfolder/config_cifar_topkacc.ini" "$nfs_home/$logs_subfolder/config_cifar_topkaccRandomAlpha.ini" "$nfs_home/$logs_subfolder/config_cifar_subsampling.ini" "$nfs_home/$logs_subfolder/config_cifar_wavelet.ini" "$nfs_home/$logs_subfolder/config_cifar_waveletRandomAlpha.ini")
#tests=("$nfs_home/$logs_subfolder/config_cifar_subsampling.ini" "$nfs_home/$logs_subfolder/config_cifar_sharing.ini" "$nfs_home/$logs_subfolder/config_cifar_waveletRandomAlpha.ini")
#tests=("$nfs_home/$logs_subfolder/config_cifar_waveletRandomAlpha.ini")
# Learning rates # Learning rates
lr="0.001" lr="0.01"
# Batch size # Batch size
batchsize="16" batchsize="8"
# The number of communication rounds per global epoch # The number of communication rounds per global epoch
comm_rounds_per_global_epoch="1" comm_rounds_per_global_epoch="20"
procs=`expr $procs_per_machine \* $machines` procs=`expr $procs_per_machine \* $machines`
echo procs: $procs echo procs: $procs
# Celeba has 63741 samples # Celeba has 63741 samples
# Reddit has 70642 # Reddit has 70642
# Femnist 734463 # Femnist 734463
# Shakespeares 3678451, subsampled 678696 # Shakespeares 3678451
# cifar 50000 dataset_size=50000
dataset_size=734463
# Calculating the number of samples that each user/proc will have on average # Calculating the number of samples that each user/proc will have on average
samples_per_user=`expr $dataset_size / $procs` samples_per_user=`expr $dataset_size / $procs`
echo samples per user: $samples_per_user echo samples per user: $samples_per_user
# random_seeds for which to rerun the experiments # random_seeds for which to rerun the experiments
random_seeds=("97") # random_seeds=("90" "91" "92" "93" "94")
random_seeds=("94")
# random_seed = 97 # random_seed = 97
echo batchsize: $batchsize echo batchsize: $batchsize
echo communication rounds per global epoch: $comm_rounds_per_global_epoch echo communication rounds per global epoch: $comm_rounds_per_global_epoch
...@@ -85,10 +87,10 @@ echo iterations: $iterations ...@@ -85,10 +87,10 @@ echo iterations: $iterations
batches_per_comm_round=$($env_python -c "from math import floor; x = floor($batches_per_epoch / $comm_rounds_per_global_epoch); print(1 if x==0 else x)") batches_per_comm_round=$($env_python -c "from math import floor; x = floor($batches_per_epoch / $comm_rounds_per_global_epoch); print(1 if x==0 else x)")
# since the batches per communication round were rounded down we need to change the number of iterations to reflect that # since the batches per communication round were rounded down we need to change the number of iterations to reflect that
new_iterations=$($env_python -c "from math import floor; tmp = floor($batches_per_epoch / $comm_rounds_per_global_epoch); x = 1 if tmp == 0 else tmp; y = floor((($batches_per_epoch / $comm_rounds_per_global_epoch)/x)*$iterations); print($iterations if y<$iterations else y)") new_iterations=$($env_python -c "from math import floor; tmp = floor($batches_per_epoch / $comm_rounds_per_global_epoch); x = 1 if tmp == 0 else tmp; y = floor((($batches_per_epoch / $comm_rounds_per_global_epoch)/x)*$iterations); print($iterations if y<$iterations else y)")
echo batches per communication round: $batches_per_comm_round
echo corrected iterations: $new_iterations
test_after=$(($new_iterations / $global_epochs)) test_after=$(($new_iterations / $global_epochs))
echo test after: $test_after echo test after: $test_after
echo batches per communication round: $batches_per_comm_round
echo corrected iterations: $new_iterations
for i in "${tests[@]}" for i in "${tests[@]}"
do do
for seed in "${random_seeds[@]}" for seed in "${random_seeds[@]}"
...@@ -96,9 +98,14 @@ do ...@@ -96,9 +98,14 @@ do
echo $i echo $i
IFS='_' read -ra NAMES <<< $i IFS='_' read -ra NAMES <<< $i
IFS='.' read -ra NAME <<< ${NAMES[-1]} IFS='.' read -ra NAME <<< ${NAMES[-1]}
log_dir=$nfs_home$logs_subfolder/${NAME[0]}:lr=$lr:r=$comm_rounds_per_global_epoch:b=$batchsize:$(date '+%Y-%m-%dT%H:%M')/machine$m #log_dir_base=$nfs_home$logs_subfolder/${NAME[0]}:lr=$lr:r=$comm_rounds_per_global_epoch:b=$batchsize:$(date '+%Y-%m-%dT%H:%M')
echo results are stored in: $log_dir log_dir_base=$nfs_home/$logs_subfolder/lr=$lr:r=$comm_rounds_per_global_epoch:b=$batchsize:$(date '+%Y-%m-%dT%H:%M')
echo results are stored in: $log_dir_base
log_dir=$log_dir_base/machine$m
mkdir -p $log_dir mkdir -p $log_dir
weight_store_dir=$log_dir_base/weights
mkdir -p $weight_store_dir
graph=$nfs_home/decentralizepy/eval/96_regular.edges
cp $i $config_file cp $i $config_file
# changing the config files to reflect the values of the current grid search state # changing the config files to reflect the values of the current grid search state
$python_bin/crudini --set $config_file COMMUNICATION addresses_filepath $ip_machines $python_bin/crudini --set $config_file COMMUNICATION addresses_filepath $ip_machines
...@@ -106,10 +113,14 @@ do ...@@ -106,10 +113,14 @@ do
$python_bin/crudini --set $config_file TRAIN_PARAMS rounds $batches_per_comm_round $python_bin/crudini --set $config_file TRAIN_PARAMS rounds $batches_per_comm_round
$python_bin/crudini --set $config_file TRAIN_PARAMS batch_size $batchsize $python_bin/crudini --set $config_file TRAIN_PARAMS batch_size $batchsize
$python_bin/crudini --set $config_file DATASET random_seed $seed $python_bin/crudini --set $config_file DATASET random_seed $seed
$python_bin/crudini --set $config_file COMMUNICATION addresses_filepath $ip_machines
$python_bin/crudini --set $config_file COMMUNICATION offset 10720
# $env_python $eval_file -cte 0 -ro 0 -tea $test_after -ld $log_dir -wsd $weight_store_dir -mid $m -ps $procs_per_machine -ms $machines -is $new_iterations -gf $graph -ta $test_after -cf $config_file -ll $log_level
$env_python $eval_file -ro 0 -tea $test_after -ld $log_dir -mid $m -ps $procs_per_machine -ms $machines -is $new_iterations -gf $graph -ta $test_after -cf $config_file -ll $log_level $env_python $eval_file -ro 0 -tea $test_after -ld $log_dir -mid $m -ps $procs_per_machine -ms $machines -is $new_iterations -gf $graph -ta $test_after -cf $config_file -ll $log_level
echo $i is done echo $i is done
sleep 200 sleep 200
echo end of sleep echo end of sleep
done done
done done
# #
\ No newline at end of file
eval/run_xtimes_celeba.sh
View file @ d140bf44
...@@ -38,11 +38,11 @@ cd $decpy_path ...@@ -38,11 +38,11 @@ cd $decpy_path
env_python=$python_bin/python3 env_python=$python_bin/python3
graph=96_regular.edges graph=96_regular.edges
config_file=~/tmp/config.ini config_file=~/tmp/config_celeba_sharing.ini
procs_per_machine=16 procs_per_machine=16
machines=6 machines=6
global_epochs=150 global_epochs=150
eval_file=testing.py eval_file=testingFederated.py
log_level=INFO log_level=INFO
ip_machines=$nfs_home/configs/ip_addr_6Machines.json ip_machines=$nfs_home/configs/ip_addr_6Machines.json
...@@ -51,7 +51,8 @@ m=`cat $ip_machines | grep $(/sbin/ifconfig ens785 | grep 'inet ' | awk '{print ...@@ -51,7 +51,8 @@ m=`cat $ip_machines | grep $(/sbin/ifconfig ens785 | grep 'inet ' | awk '{print
export PYTHONFAULTHANDLER=1 export PYTHONFAULTHANDLER=1
# Base configs for which the gird search is done # Base configs for which the gird search is done
tests=("step_configs/config_celeba_sharing.ini" "step_configs/config_celeba_partialmodel.ini" "step_configs/config_celeba_topkacc.ini" "step_configs/config_celeba_subsampling.ini" "step_configs/config_celeba_wavelet.ini") #tests=("step_configs/config_celeba_sharing.ini" "step_configs/config_celeba_partialmodel.ini" "step_configs/config_celeba_topkacc.ini" "step_configs/config_celeba_subsampling.ini" "step_configs/config_celeba_wavelet.ini")
tests=("step_configs/config_celeba_sharing.ini")
# Learning rates # Learning rates
lr="0.001" lr="0.001"
# Batch size # Batch size
...@@ -66,7 +67,8 @@ samples_per_user=`expr $dataset_size / $procs` ...@@ -66,7 +67,8 @@ samples_per_user=`expr $dataset_size / $procs`
echo samples per user: $samples_per_user echo samples per user: $samples_per_user
# random_seeds for which to rerun the experiments # random_seeds for which to rerun the experiments
random_seeds=("90" "91" "92" "93" "94") #random_seeds=("90" "91" "92" "93" "94")
random_seeds=("90")
# random_seed = 97 # random_seed = 97
echo batchsize: $batchsize echo batchsize: $batchsize
echo communication rounds per global epoch: $comm_rounds_per_global_epoch echo communication rounds per global epoch: $comm_rounds_per_global_epoch
...@@ -91,7 +93,7 @@ do ...@@ -91,7 +93,7 @@ do
echo $i echo $i
IFS='_' read -ra NAMES <<< $i IFS='_' read -ra NAMES <<< $i
IFS='.' read -ra NAME <<< ${NAMES[-1]} IFS='.' read -ra NAME <<< ${NAMES[-1]}
log_dir_base=$nfs_home$logs_subfolder/${NAME[0]}:lr=$lr:r=$comm_rounds_per_global_epoch:b=$batchsize:$(date '+%Y-%m-%dT%H:%M') log_dir_base=$nfs_home/$logs_subfolder/${NAME[0]}:lr=$lr:r=$comm_rounds_per_global_epoch:b=$batchsize:$(date '+%Y-%m-%dT%H:%M')
echo results are stored in: $log_dir_base echo results are stored in: $log_dir_base
log_dir=$log_dir_base/machine$m log_dir=$log_dir_base/machine$m
mkdir -p $log_dir mkdir -p $log_dir
......
eval/run_xtimes_cifar.sh
View file @ d140bf44
...@@ -41,9 +41,10 @@ graph=96_regular.edges ...@@ -41,9 +41,10 @@ graph=96_regular.edges
config_file=~/tmp/config.ini config_file=~/tmp/config.ini
procs_per_machine=16 procs_per_machine=16
machines=6 machines=6
global_epochs=300 global_epochs=100
eval_file=testing.py eval_file=testingFederated.py
log_level=INFO log_level=INFO
working_rate=0.1
ip_machines=$nfs_home/configs/ip_addr_6Machines.json ip_machines=$nfs_home/configs/ip_addr_6Machines.json
...@@ -51,7 +52,7 @@ m=`cat $ip_machines | grep $(/sbin/ifconfig ens785 | grep 'inet ' | awk '{print ...@@ -51,7 +52,7 @@ m=`cat $ip_machines | grep $(/sbin/ifconfig ens785 | grep 'inet ' | awk '{print
export PYTHONFAULTHANDLER=1 export PYTHONFAULTHANDLER=1
# Base configs for which the gird search is done # Base configs for which the gird search is done
tests=("step_configs/config_cifar_sharing.ini" "step_configs/config_cifar_partialmodel.ini" "step_configs/config_cifar_topkacc.ini" "step_configs/config_cifar_subsampling.ini" "step_configs/config_cifar_wavelet.ini") tests=("step_configs/config_cifar_sharing.ini")
# Learning rates # Learning rates
lr="0.01" lr="0.01"
# Batch size # Batch size
...@@ -66,7 +67,7 @@ samples_per_user=`expr $dataset_size / $procs` ...@@ -66,7 +67,7 @@ samples_per_user=`expr $dataset_size / $procs`
echo samples per user: $samples_per_user echo samples per user: $samples_per_user
# random_seeds for which to rerun the experiments # random_seeds for which to rerun the experiments
random_seeds=("90" "91" "92" "93" "94") random_seeds=("90")
# random_seed = 97 # random_seed = 97
echo batchsize: $batchsize echo batchsize: $batchsize
echo communication rounds per global epoch: $comm_rounds_per_global_epoch echo communication rounds per global epoch: $comm_rounds_per_global_epoch
...@@ -91,7 +92,7 @@ do ...@@ -91,7 +92,7 @@ do
echo $i echo $i
IFS='_' read -ra NAMES <<< $i IFS='_' read -ra NAMES <<< $i
IFS='.' read -ra NAME <<< ${NAMES[-1]} IFS='.' read -ra NAME <<< ${NAMES[-1]}
log_dir_base=$nfs_home$logs_subfolder/${NAME[0]}:lr=$lr:r=$comm_rounds_per_global_epoch:b=$batchsize:$(date '+%Y-%m-%dT%H:%M') log_dir_base=$nfs_home/$logs_subfolder/${NAME[0]}:lr=$lr:r=$comm_rounds_per_global_epoch:b=$batchsize:$(date '+%Y-%m-%dT%H:%M')
echo results are stored in: $log_dir_base echo results are stored in: $log_dir_base
log_dir=$log_dir_base/machine$m log_dir=$log_dir_base/machine$m
mkdir -p $log_dir mkdir -p $log_dir
...@@ -104,7 +105,7 @@ do ...@@ -104,7 +105,7 @@ do
$python_bin/crudini --set $config_file TRAIN_PARAMS rounds $batches_per_comm_round $python_bin/crudini --set $config_file TRAIN_PARAMS rounds $batches_per_comm_round
$python_bin/crudini --set $config_file TRAIN_PARAMS batch_size $batchsize $python_bin/crudini --set $config_file TRAIN_PARAMS batch_size $batchsize
$python_bin/crudini --set $config_file DATASET random_seed $seed $python_bin/crudini --set $config_file DATASET random_seed $seed
$env_python $eval_file -ro 0 -tea $test_after -ld $log_dir -wsd $weight_store_dir -mid $m -ps $procs_per_machine -ms $machines -is $new_iterations -gf $graph -ta $test_after -cf $config_file -ll $log_level $env_python $eval_file -ro 0 -tea $test_after -ld $log_dir -wsd $weight_store_dir -mid $m -ps $procs_per_machine -ms $machines -is $new_iterations -gf $graph -ta $test_after -cf $config_file -ll $log_level -wr $working_rate
echo $i is done echo $i is done
sleep 200 sleep 200
echo end of sleep echo end of sleep
......
eval/run_xtimes_femnist.sh
View file @ d140bf44
...@@ -38,11 +38,11 @@ cd $decpy_path ...@@ -38,11 +38,11 @@ cd $decpy_path
env_python=$python_bin/python3 env_python=$python_bin/python3
graph=96_regular.edges graph=96_regular.edges
config_file=~/tmp/config.ini config_file=~/tmp/config_femnist_sharing.ini
procs_per_machine=16 procs_per_machine=16
machines=6 machines=6
global_epochs=80 global_epochs=80
eval_file=testing.py eval_file=testingFederated.py
log_level=INFO log_level=INFO
ip_machines=$nfs_home/configs/ip_addr_6Machines.json ip_machines=$nfs_home/configs/ip_addr_6Machines.json
...@@ -51,7 +51,8 @@ m=`cat $ip_machines | grep $(/sbin/ifconfig ens785 | grep 'inet ' | awk '{print ...@@ -51,7 +51,8 @@ m=`cat $ip_machines | grep $(/sbin/ifconfig ens785 | grep 'inet ' | awk '{print
export PYTHONFAULTHANDLER=1 export PYTHONFAULTHANDLER=1
# Base configs for which the gird search is done # Base configs for which the gird search is done
tests=("step_configs/config_femnist_sharing.ini" "step_configs/config_femnist_partialmodel.ini" "step_configs/config_femnist_topkacc.ini" "step_configs/config_femnist_subsampling.ini" "step_configs/config_femnist_wavelet.ini") #tests=("step_configs/config_femnist_sharing.ini" "step_configs/config_femnist_partialmodel.ini" "step_configs/config_femnist_topkacc.ini" "step_configs/config_femnist_subsampling.ini" "step_configs/config_femnist_wavelet.ini")
tests=("step_configs/config_femnist_sharing.ini")
# Learning rates # Learning rates
lr="0.01" lr="0.01"
# Batch size # Batch size
...@@ -66,7 +67,8 @@ samples_per_user=`expr $dataset_size / $procs` ...@@ -66,7 +67,8 @@ samples_per_user=`expr $dataset_size / $procs`
echo samples per user: $samples_per_user echo samples per user: $samples_per_user
# random_seeds for which to rerun the experiments # random_seeds for which to rerun the experiments
random_seeds=("90" "91" "92" "93" "94") #random_seeds=("90" "91" "92" "93" "94")
random_seeds=("90")
# random_seed = 97 # random_seed = 97
echo batchsize: $batchsize echo batchsize: $batchsize
echo communication rounds per global epoch: $comm_rounds_per_global_epoch echo communication rounds per global epoch: $comm_rounds_per_global_epoch
...@@ -91,7 +93,7 @@ do ...@@ -91,7 +93,7 @@ do
echo $i echo $i
IFS='_' read -ra NAMES <<< $i IFS='_' read -ra NAMES <<< $i
IFS='.' read -ra NAME <<< ${NAMES[-1]} IFS='.' read -ra NAME <<< ${NAMES[-1]}
log_dir_base=$nfs_home$logs_subfolder/${NAME[0]}:lr=$lr:r=$comm_rounds_per_global_epoch:b=$batchsize:$(date '+%Y-%m-%dT%H:%M') log_dir_base=$nfs_home/$logs_subfolder/${NAME[0]}:lr=$lr:r=$comm_rounds_per_global_epoch:b=$batchsize:$(date '+%Y-%m-%dT%H:%M')
echo results are stored in: $log_dir_base echo results are stored in: $log_dir_base
log_dir=$log_dir_base/machine$m log_dir=$log_dir_base/machine$m
mkdir -p $log_dir mkdir -p $log_dir
......
eval/run_xtimes_reddit.sh
View file @ d140bf44
...@@ -38,11 +38,11 @@ cd $decpy_path ...@@ -38,11 +38,11 @@ cd $decpy_path
env_python=$python_bin/python3 env_python=$python_bin/python3
graph=96_regular.edges graph=96_regular.edges
config_file=~/tmp/config.ini config_file=~/tmp/config_reddit_sharing.ini
procs_per_machine=16 procs_per_machine=16
machines=6 machines=6
global_epochs=50 global_epochs=50
eval_file=testing.py eval_file=testingFederated.py
log_level=INFO log_level=INFO
ip_machines=$nfs_home/configs/ip_addr_6Machines.json ip_machines=$nfs_home/configs/ip_addr_6Machines.json
...@@ -51,7 +51,8 @@ m=`cat $ip_machines | grep $(/sbin/ifconfig ens785 | grep 'inet ' | awk '{print ...@@ -51,7 +51,8 @@ m=`cat $ip_machines | grep $(/sbin/ifconfig ens785 | grep 'inet ' | awk '{print
export PYTHONFAULTHANDLER=1 export PYTHONFAULTHANDLER=1
# Base configs for which the gird search is done # Base configs for which the gird search is done
tests=("step_configs/config_reddit_sharing.ini" "step_configs/config_reddit_partialmodel.ini" "step_configs/config_reddit_topkacc.ini" "step_configs/config_reddit_subsampling.ini" "step_configs/config_reddit_wavelet.ini") #tests=("step_configs/config_reddit_sharing.ini" "step_configs/config_reddit_partialmodel.ini" "step_configs/config_reddit_topkacc.ini" "step_configs/config_reddit_subsampling.ini" "step_configs/config_reddit_wavelet.ini")
tests=("step_configs/config_reddit_sharing.ini")
# Learning rates # Learning rates
lr="1" lr="1"
# Batch size # Batch size
...@@ -66,7 +67,8 @@ samples_per_user=`expr $dataset_size / $procs` ...@@ -66,7 +67,8 @@ samples_per_user=`expr $dataset_size / $procs`
echo samples per user: $samples_per_user echo samples per user: $samples_per_user
# random_seeds for which to rerun the experiments # random_seeds for which to rerun the experiments
random_seeds=("90" "91" "92" "93" "94") #random_seeds=("90" "91" "92" "93" "94")
random_seeds=("90")
# random_seed = 97 # random_seed = 97
echo batchsize: $batchsize echo batchsize: $batchsize
echo communication rounds per global epoch: $comm_rounds_per_global_epoch echo communication rounds per global epoch: $comm_rounds_per_global_epoch
...@@ -91,7 +93,7 @@ do ...@@ -91,7 +93,7 @@ do
echo $i echo $i
IFS='_' read -ra NAMES <<< $i IFS='_' read -ra NAMES <<< $i
IFS='.' read -ra NAME <<< ${NAMES[-1]} IFS='.' read -ra NAME <<< ${NAMES[-1]}
log_dir_base=$nfs_home$logs_subfolder/${NAME[0]}:lr=$lr:r=$comm_rounds_per_global_epoch:b=$batchsize:$(date '+%Y-%m-%dT%H:%M') log_dir_base=$nfs_home/$logs_subfolder/${NAME[0]}:lr=$lr:r=$comm_rounds_per_global_epoch:b=$batchsize:$(date '+%Y-%m-%dT%H:%M')
echo results are stored in: $log_dir_base echo results are stored in: $log_dir_base
log_dir=$log_dir_base/machine$m log_dir=$log_dir_base/machine$m
mkdir -p $log_dir mkdir -p $log_dir
......
eval/run_xtimes_shakespeare.sh
View file @ d140bf44
...@@ -38,11 +38,11 @@ cd $decpy_path ...@@ -38,11 +38,11 @@ cd $decpy_path
env_python=$python_bin/python3 env_python=$python_bin/python3
graph=96_regular.edges graph=96_regular.edges
config_file=~/tmp/config.ini config_file=~/tmp/config_shakespeare_sharing.ini
procs_per_machine=16 procs_per_machine=16
machines=6 machines=6
global_epochs=200 global_epochs=100
eval_file=testing.py eval_file=testingFederated.py
log_level=INFO log_level=INFO
ip_machines=$nfs_home/configs/ip_addr_6Machines.json ip_machines=$nfs_home/configs/ip_addr_6Machines.json
...@@ -51,7 +51,8 @@ m=`cat $ip_machines | grep $(/sbin/ifconfig ens785 | grep 'inet ' | awk '{print ...@@ -51,7 +51,8 @@ m=`cat $ip_machines | grep $(/sbin/ifconfig ens785 | grep 'inet ' | awk '{print
export PYTHONFAULTHANDLER=1 export PYTHONFAULTHANDLER=1
# Base configs for which the gird search is done # Base configs for which the gird search is done
tests=("step_configs/config_shakespeare_sharing.ini" "step_configs/config_shakespeare_partialmodel.ini" "step_configs/config_shakespeare_topkacc.ini" "step_configs/config_shakespeare_subsampling.ini" "step_configs/config_shakespeare_wavelet.ini") #tests=("step_configs/config_shakespeare_sharing.ini" "step_configs/config_shakespeare_partialmodel.ini" "step_configs/config_shakespeare_topkacc.ini" "step_configs/config_shakespeare_subsampling.ini" "step_configs/config_shakespeare_wavelet.ini")
tests=("step_configs/config_shakespeare_sharing.ini")
# Learning rates # Learning rates
lr="0.5" lr="0.5"
# Batch size # Batch size
...@@ -66,7 +67,8 @@ samples_per_user=`expr $dataset_size / $procs` ...@@ -66,7 +67,8 @@ samples_per_user=`expr $dataset_size / $procs`
echo samples per user: $samples_per_user echo samples per user: $samples_per_user
# random_seeds for which to rerun the experiments # random_seeds for which to rerun the experiments
random_seeds=("90" "91" "92" "93" "94") #random_seeds=("90" "91" "92" "93" "94")
random_seeds=("90")
# random_seed = 97 # random_seed = 97
echo batchsize: $batchsize echo batchsize: $batchsize
echo communication rounds per global epoch: $comm_rounds_per_global_epoch echo communication rounds per global epoch: $comm_rounds_per_global_epoch
...@@ -91,7 +93,7 @@ do ...@@ -91,7 +93,7 @@ do
echo $i echo $i
IFS='_' read -ra NAMES <<< $i IFS='_' read -ra NAMES <<< $i
IFS='.' read -ra NAME <<< ${NAMES[-1]} IFS='.' read -ra NAME <<< ${NAMES[-1]}
log_dir_base=$nfs_home$logs_subfolder/${NAME[0]}:lr=$lr:r=$comm_rounds_per_global_epoch:b=$batchsize:$(date '+%Y-%m-%dT%H:%M') log_dir_base=$nfs_home/$logs_subfolder/${NAME[0]}:lr=$lr:r=$comm_rounds_per_global_epoch:b=$batchsize:$(date '+%Y-%m-%dT%H:%M')
echo results are stored in: $log_dir_base echo results are stored in: $log_dir_base
log_dir=$log_dir_base/machine$m log_dir=$log_dir_base/machine$m
mkdir -p $log_dir mkdir -p $log_dir
......
eval/testing.py
View file @ d140bf44
...@@ -8,7 +8,7 @@ from torch import multiprocessing as mp ...@@ -8,7 +8,7 @@ from torch import multiprocessing as mp
from decentralizepy import utils from decentralizepy import utils
from decentralizepy.graphs.Graph import Graph from decentralizepy.graphs.Graph import Graph
from decentralizepy.mappings.Linear import Linear from decentralizepy.mappings.Linear import Linear
from decentralizepy.node.Node import Node from decentralizepy.node.DPSGDNode import DPSGDNode
def read_ini(file_path): def read_ini(file_path):
...@@ -50,22 +50,30 @@ if __name__ == "__main__": ...@@ -50,22 +50,30 @@ if __name__ == "__main__":
l = Linear(n_machines, procs_per_machine) l = Linear(n_machines, procs_per_machine)
m_id = args.machine_id m_id = args.machine_id
mp.spawn( processes = []
fn=Node, for r in range(procs_per_machine):
nprocs=procs_per_machine, processes.append(
args=[ mp.Process(
m_id, target=DPSGDNode,
l, args=[
g, r,
my_config, m_id,
args.iterations, l,
args.log_dir, g,
args.weights_store_dir, my_config,
log_level[args.log_level], args.iterations,
args.test_after, args.log_dir,
args.train_evaluate_after, args.weights_store_dir,
args.reset_optimizer, log_level[args.log_level],
args.centralized_train_eval, args.test_after,
args.centralized_test_eval, args.train_evaluate_after,
], args.reset_optimizer,
) ],
)
)
for p in processes:
p.start()
for p in processes:
p.join()
eval/testingFederated.py 0 → 100644
View file @ d140bf44
import logging
from pathlib import Path
from shutil import copy
from localconfig import LocalConfig
from torch import multiprocessing as mp
from decentralizepy import utils
from decentralizepy.graphs.Graph import Graph
from decentralizepy.mappings.Linear import Linear
from decentralizepy.node.DPSGDNodeFederated import DPSGDNodeFederated
from decentralizepy.node.FederatedParameterServer import FederatedParameterServer
def read_ini(file_path):
config = LocalConfig(file_path)
for section in config:
print("Section: ", section)
for key, value in config.items(section):
print((key, value))
print(dict(config.items("DATASET")))
return config
if __name__ == "__main__":
args = utils.get_args()
Path(args.log_dir).mkdir(parents=True, exist_ok=True)
log_level = {
"INFO": logging.INFO,
"DEBUG": logging.DEBUG,
"WARNING": logging.WARNING,
"ERROR": logging.ERROR,
"CRITICAL": logging.CRITICAL,
}
config = read_ini(args.config_file)
my_config = dict()
for section in config:
my_config[section] = dict(config.items(section))
copy(args.config_file, args.log_dir)
copy(args.graph_file, args.log_dir)
utils.write_args(args, args.log_dir)
g = Graph()
g.read_graph_from_file(args.graph_file, args.graph_type)
n_machines = args.machines
procs_per_machine = args.procs_per_machine
l = Linear(n_machines, procs_per_machine)
m_id = args.machine_id
sm = args.server_machine
sr = args.server_rank
processes = []
if sm == m_id:
processes.append(
mp.Process(
target=FederatedParameterServer,
args=[
sr,
m_id,
l,
g,
my_config,
args.iterations,
args.log_dir,
args.weights_store_dir,
log_level[args.log_level],
args.test_after,
args.train_evaluate_after,
args.working_rate,
],
)
)
for r in range(0, procs_per_machine):
processes.append(
mp.Process(
target=DPSGDNodeFederated,
args=[
r,
m_id,
l,
g,
my_config,
args.iterations,
args.log_dir,
args.weights_store_dir,
log_level[args.log_level],
args.test_after,
args.train_evaluate_after,
args.reset_optimizer,
],
)
)
for p in processes:
p.start()
for p in processes:
p.join()
eval/testingKNN.py 0 → 100644
View file @ d140bf44
import logging
from pathlib import Path
from shutil import copy
from localconfig import LocalConfig
from torch import multiprocessing as mp
from decentralizepy import utils
from decentralizepy.graphs.Graph import Graph
from decentralizepy.mappings.Linear import Linear
from decentralizepy.node.KNN import KNN
def read_ini(file_path):
config = LocalConfig(file_path)
for section in config:
print("Section: ", section)
for key, value in config.items(section):
print((key, value))
print(dict(config.items("DATASET")))
return config
if __name__ == "__main__":
args = utils.get_args()
Path(args.log_dir).mkdir(parents=True, exist_ok=True)
log_level = {
"INFO": logging.INFO,
"DEBUG": logging.DEBUG,
"WARNING": logging.WARNING,
"ERROR": logging.ERROR,
"CRITICAL": logging.CRITICAL,
}
config = read_ini(args.config_file)
my_config = dict()
for section in config:
my_config[section] = dict(config.items(section))
copy(args.config_file, args.log_dir)
copy(args.graph_file, args.log_dir)
utils.write_args(args, args.log_dir)
g = Graph()
g.read_graph_from_file(args.graph_file, args.graph_type)
n_machines = args.machines
procs_per_machine = args.procs_per_machine
l = Linear(n_machines, procs_per_machine)
m_id = args.machine_id
processes = []
for r in range(procs_per_machine):
processes.append(
mp.Process(
target=KNN,
args=[
r,
m_id,
l,
g,
my_config,
args.iterations,
args.log_dir,
args.weights_store_dir,
log_level[args.log_level],
args.test_after,
args.train_evaluate_after,
args.reset_optimizer,
],
)
)
for p in processes:
p.start()
for p in processes:
p.join()
eval/testingPeerSampler.py 0 → 100644
View file @ d140bf44
import logging
from pathlib import Path
from shutil import copy
from localconfig import LocalConfig
from torch import multiprocessing as mp
from decentralizepy import utils
from decentralizepy.graphs.Graph import Graph
from decentralizepy.mappings.Linear import Linear
from decentralizepy.node.DPSGDWithPeerSampler import DPSGDWithPeerSampler
from decentralizepy.node.PeerSampler import PeerSampler
def read_ini(file_path):
config = LocalConfig(file_path)
for section in config:
print("Section: ", section)
for key, value in config.items(section):
print((key, value))
print(dict(config.items("DATASET")))
return config
if __name__ == "__main__":
args = utils.get_args()
Path(args.log_dir).mkdir(parents=True, exist_ok=True)
log_level = {
"INFO": logging.INFO,
"DEBUG": logging.DEBUG,
"WARNING": logging.WARNING,
"ERROR": logging.ERROR,
"CRITICAL": logging.CRITICAL,
}
config = read_ini(args.config_file)
my_config = dict()
for section in config:
my_config[section] = dict(config.items(section))
copy(args.config_file, args.log_dir)
copy(args.graph_file, args.log_dir)
utils.write_args(args, args.log_dir)
g = Graph()
g.read_graph_from_file(args.graph_file, args.graph_type)
n_machines = args.machines
procs_per_machine = args.procs_per_machine
l = Linear(n_machines, procs_per_machine)
m_id = args.machine_id
sm = args.server_machine
sr = args.server_rank
processes = []
if sm == m_id:
processes.append(
mp.Process(
# target=PeerSamplerDynamic,
target=PeerSampler,
args=[
sr,
m_id,
l,
g,
my_config,
args.iterations,
args.log_dir,
log_level[args.log_level],
],
)
)
for r in range(0, procs_per_machine):
processes.append(
mp.Process(
target=DPSGDWithPeerSampler,
args=[
r,
m_id,
l,
g,
my_config,
args.iterations,
args.log_dir,
args.weights_store_dir,
log_level[args.log_level],
args.test_after,
args.train_evaluate_after,
args.reset_optimizer,
],
)
)
for p in processes:
p.start()
for p in processes:
p.join()
eval/testingPeerSamplerDynamic.py 0 → 100644
View file @ d140bf44
import logging
from pathlib import Path
from shutil import copy
from localconfig import LocalConfig
from torch import multiprocessing as mp
from decentralizepy import utils
from decentralizepy.graphs.Graph import Graph
from decentralizepy.mappings.Linear import Linear
from decentralizepy.node.DPSGDWithPeerSampler import DPSGDWithPeerSampler
from decentralizepy.node.PeerSamplerDynamic import PeerSamplerDynamic
def read_ini(file_path):
config = LocalConfig(file_path)
for section in config:
print("Section: ", section)
for key, value in config.items(section):
print((key, value))
print(dict(config.items("DATASET")))
return config
if __name__ == "__main__":
args = utils.get_args()
Path(args.log_dir).mkdir(parents=True, exist_ok=True)
log_level = {
"INFO": logging.INFO,
"DEBUG": logging.DEBUG,
"WARNING": logging.WARNING,
"ERROR": logging.ERROR,
"CRITICAL": logging.CRITICAL,
}
config = read_ini(args.config_file)
my_config = dict()
for section in config:
my_config[section] = dict(config.items(section))
copy(args.config_file, args.log_dir)
copy(args.graph_file, args.log_dir)
utils.write_args(args, args.log_dir)
g = Graph()
g.read_graph_from_file(args.graph_file, args.graph_type)
n_machines = args.machines
procs_per_machine = args.procs_per_machine
l = Linear(n_machines, procs_per_machine)
m_id = args.machine_id
sm = args.server_machine
sr = args.server_rank
processes = []
if sm == m_id:
processes.append(
mp.Process(
target=PeerSamplerDynamic,
args=[
sr,
m_id,
l,
g,
my_config,
args.iterations,
args.log_dir,
log_level[args.log_level],
],
)
)
for r in range(0, procs_per_machine):
processes.append(
mp.Process(
target=DPSGDWithPeerSampler,
args=[
r,
m_id,
l,
g,
my_config,
args.iterations,
args.log_dir,
args.weights_store_dir,
log_level[args.log_level],
args.test_after,
args.train_evaluate_after,
args.reset_optimizer,
],
)
)
for p in processes:
p.start()
for p in processes:
p.join()
setup.cfg
View file @ d140bf44
...@@ -54,7 +54,7 @@ python_requires = >=3.6 ...@@ -54,7 +54,7 @@ python_requires = >=3.6
where = src where = src
[options.extras_require] [options.extras_require]
dev = dev =
black black>22.3.0
coverage coverage
isort isort
pytest pytest
......
src/decentralizepy/communication/TCP.py
View file @ d140bf44
import importlib
import json import json
import logging import logging
import pickle import pickle
...@@ -36,7 +35,8 @@ class TCP(Communication): ...@@ -36,7 +35,8 @@ class TCP(Communication):
""" """
machine_addr = self.ip_addrs[str(machine_id)] machine_addr = self.ip_addrs[str(machine_id)]
port = rank + self.offset port = (2 * rank + 1) + self.offset
assert port > 0
return "tcp://{}:{}".format(machine_addr, port) return "tcp://{}:{}".format(machine_addr, port)
def __init__( def __init__(
...@@ -46,10 +46,8 @@ class TCP(Communication): ...@@ -46,10 +46,8 @@ class TCP(Communication):
mapping, mapping,
total_procs, total_procs,
addresses_filepath, addresses_filepath,
compress=False, offset=9000,
offset=20000, recv_timeout=50,
compression_package=None,
compression_class=None,
): ):
""" """
Constructor Constructor
...@@ -81,30 +79,22 @@ class TCP(Communication): ...@@ -81,30 +79,22 @@ class TCP(Communication):
self.rank = rank self.rank = rank
self.machine_id = machine_id self.machine_id = machine_id
self.mapping = mapping self.mapping = mapping
self.offset = 20000 + offset self.offset = offset
self.recv_timeout = recv_timeout
self.uid = mapping.get_uid(rank, machine_id) self.uid = mapping.get_uid(rank, machine_id)
self.identity = str(self.uid).encode() self.identity = str(self.uid).encode()
self.context = zmq.Context() self.context = zmq.Context()
self.router = self.context.socket(zmq.ROUTER) self.router = self.context.socket(zmq.ROUTER)
self.router.setsockopt(zmq.IDENTITY, self.identity) self.router.setsockopt(zmq.IDENTITY, self.identity)
self.router.setsockopt(zmq.RCVTIMEO, self.recv_timeout)
self.router.setsockopt(zmq.ROUTER_MANDATORY, 1)
self.router.bind(self.addr(rank, machine_id)) self.router.bind(self.addr(rank, machine_id))
self.sent_disconnections = False
self.compress = compress
if compression_package and compression_class:
compressor_module = importlib.import_module(compression_package)
compressor_class = getattr(compressor_module, compression_class)
self.compressor = compressor_class()
logging.info(f"Using the {compressor_class} to compress the data")
else:
assert not self.compress
self.total_data = 0 self.total_data = 0
self.total_meta = 0 self.total_meta = 0
self.peer_deque = deque() self.peer_deque = deque()
self.peer_sockets = dict() self.peer_sockets = dict()
self.barrier = set()
def __del__(self): def __del__(self):
""" """
...@@ -128,26 +118,12 @@ class TCP(Communication): ...@@ -128,26 +118,12 @@ class TCP(Communication):
Encoded data Encoded data
""" """
if self.compress: data_len = 0
if "indices" in data: if "params" in data:
data["indices"] = self.compressor.compress(data["indices"])
assert "params" in data
data["params"] = self.compressor.compress_float(data["params"])
data_len = len(pickle.dumps(data["params"])) data_len = len(pickle.dumps(data["params"]))
output = pickle.dumps(data) output = pickle.dumps(data)
self.total_meta += len(output) - data_len
# the compressed meta data gets only a few bytes smaller after pickling self.total_data += data_len
self.total_meta += len(output) - data_len
self.total_data += data_len
else:
output = pickle.dumps(data)
# centralized testing uses its own instance
if type(data) == dict:
assert "params" in data
data_len = len(pickle.dumps(data["params"]))
self.total_meta += len(output) - data_len
self.total_data += data_len
return output return output
def decrypt(self, sender, data): def decrypt(self, sender, data):
...@@ -168,65 +144,37 @@ class TCP(Communication): ...@@ -168,65 +144,37 @@ class TCP(Communication):
""" """
sender = int(sender.decode()) sender = int(sender.decode())
if self.compress: data = pickle.loads(data)
data = pickle.loads(data)
if "indices" in data:
data["indices"] = self.compressor.decompress(data["indices"])
if "params" in data:
data["params"] = self.compressor.decompress_float(data["params"])
else:
data = pickle.loads(data)
return sender, data return sender, data
def connect_neighbors(self, neighbors): def init_connection(self, neighbor):
""" """
Connects all neighbors. Sends HELLO. Waits for HELLO. Initiates a socket to a given node.
Caches any data received while waiting for HELLOs.
Parameters Parameters
---------- ----------
neighbors : list(int) neighbor : int
List of neighbors neighbor to connect to
Raises
------
RuntimeError
If received BYE while waiting for HELLO
""" """
logging.info("Sending connection request to neighbors") logging.debug("Connecting to my neighbour: {}".format(neighbor))
for uid in neighbors: id = str(neighbor).encode()
logging.debug("Connecting to my neighbour: {}".format(uid)) req = self.context.socket(zmq.DEALER)
id = str(uid).encode() req.setsockopt(zmq.IDENTITY, self.identity)
req = self.context.socket(zmq.DEALER) req.connect(self.addr(*self.mapping.get_machine_and_rank(neighbor)))
req.setsockopt(zmq.IDENTITY, self.identity) self.peer_sockets[id] = req
req.connect(self.addr(*self.mapping.get_machine_and_rank(uid)))
self.peer_sockets[id] = req def destroy_connection(self, neighbor, linger=None):
req.send(HELLO) id = str(neighbor).encode()
if self.already_connected(neighbor):
num_neighbors = len(neighbors) self.peer_sockets[id].close(linger=linger)
while len(self.barrier) < num_neighbors: del self.peer_sockets[id]
sender, recv = self.router.recv_multipart()
def already_connected(self, neighbor):
if recv == HELLO: id = str(neighbor).encode()
logging.debug("Received {} from {}".format(HELLO, sender)) return id in self.peer_sockets
self.barrier.add(sender)
elif recv == BYE: def receive(self, block=True):
logging.debug("Received {} from {}".format(BYE, sender))
raise RuntimeError(
"A neighbour wants to disconnect before training started!"
)
else:
logging.debug(
"Received message from {} @ connect_neighbors".format(sender)
)
self.peer_deque.append(self.decrypt(sender, recv))
logging.info("Connected to all neighbors")
self.initialized = True
def receive(self):
""" """
Returns ONE message received. Returns ONE message received.
...@@ -241,25 +189,19 @@ class TCP(Communication): ...@@ -241,25 +189,19 @@ class TCP(Communication):
If received HELLO If received HELLO
""" """
assert self.initialized == True while True:
if len(self.peer_deque) != 0: try:
resp = self.peer_deque.popleft() sender, recv = self.router.recv_multipart()
return resp s, r = self.decrypt(sender, recv)
return s, r
sender, recv = self.router.recv_multipart() except zmq.ZMQError as exc:
if exc.errno == zmq.EAGAIN:
if recv == HELLO: if not block:
logging.debug("Received {} from {}".format(HELLO, sender)) return None
raise RuntimeError( else:
"A neighbour wants to connect when everyone is connected!" continue
) else:
elif recv == BYE: raise
logging.debug("Received {} from {}".format(BYE, sender))
self.barrier.remove(sender)
return self.receive()
else:
logging.debug("Received message from {}".format(sender))
return self.decrypt(sender, recv)
def send(self, uid, data, encrypt=True): def send(self, uid, data, encrypt=True):
""" """
...@@ -273,7 +215,6 @@ class TCP(Communication): ...@@ -273,7 +215,6 @@ class TCP(Communication):
Message as a Python dictionary Message as a Python dictionary
""" """
assert self.initialized == True
if encrypt: if encrypt:
to_send = self.encrypt(data) to_send = self.encrypt(data)
else: else:
...@@ -283,28 +224,4 @@ class TCP(Communication): ...@@ -283,28 +224,4 @@ class TCP(Communication):
id = str(uid).encode() id = str(uid).encode()
self.peer_sockets[id].send(to_send) self.peer_sockets[id].send(to_send)
logging.debug("{} sent the message to {}.".format(self.uid, uid)) logging.debug("{} sent the message to {}.".format(self.uid, uid))
logging.info("Sent this round: {}".format(data_size)) logging.info("Sent message size: {}".format(data_size))
def disconnect_neighbors(self):
"""
Disconnects all neighbors.
"""
assert self.initialized == True
if not self.sent_disconnections:
logging.info("Disconnecting neighbors")
for sock in self.peer_sockets.values():
sock.send(BYE)
self.sent_disconnections = True
while len(self.barrier):
sender, recv = self.router.recv_multipart()
if recv == BYE:
logging.debug("Received {} from {}".format(BYE, sender))
self.barrier.remove(sender)
else:
logging.critical(
"Received unexpected {} from {}".format(recv, sender)
)
raise RuntimeError(
"Received a message when expecting BYE from {}".format(sender)
)
src/decentralizepy/compression/Compression.py
View file @ d140bf44
import numpy as np
class Compression: class Compression:
""" """
Compression API Compression API
......