D2L-CH5-Deep Learning Computation
因為TF2版本實做舊版的電子書內容,因此以下章節編排用Ch4命名.
https://trickygo.github.io/Dive-into-DL-TensorFlow2.0/#/
Deep Learning Computation
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"
import numpy as np
import tensorflow as tf
print(tf.__version__)
assert int(tf.__version__[0]) == 2
2.0.0
print(tf.test.is_gpu_available())
True
4.1 模型构造
4.1.1 build model from block
class MLP(tf.keras.Model):
def __init__(self):
super().__init__()
self.flatten = tf.keras.layers.Flatten() # Flatten层将除第一维（batch_size）以外的维度展平
self.dense1 = tf.keras.layers.Dense(units=256, activation=tf.nn.relu)
self.dense2 = tf.keras.layers.Dense(units=10)
def call(self, inputs):
x = self.flatten(inputs)
print(x.shape)
x = self.dense1(x)
print(x.shape)
output = self.dense2(x)
return output
X = tf.random.uniform((2,20))
net = MLP()
net(X)
(2, 20)
(2, 256)
<tf.Tensor: id=62, shape=(2, 10), dtype=float32, numpy=
array([[ 0.84096867, 0.10880844, -0.19298258, 0.09622616, -0.3129447 ,
0.05816094, 0.01181953, -0.36879984, 0.07114434, -0.23152365],
[ 0.33144152, 0.04583085, -0.27655935, 0.04085173, -0.10101503,
0.00194229, 0.03036133, -0.2423382 , 0.05986395, 0.00838697]],
dtype=float32)>
4.1.2 Sequential
model = tf.keras.models.Sequential([
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(256, activation=tf.nn.relu),
tf.keras.layers.Dense(10),
])
model(X)
<tf.Tensor: id=117, shape=(2, 10), dtype=float32, numpy=
array([[-0.09959396, -0.45667994, -0.31671545, 0.38400424, 0.29135102,
-0.1833095 , -0.09414536, -0.53436816, -0.00413986, -0.12974967],
[-0.14006008, -0.41170198, -0.04489394, 0.24267752, 0.22950622,
-0.03381499, 0.03787727, -0.32568848, 0.04253128, -0.27896702]],
dtype=float32)>
4.1.3 build complex model
class FancyMLP(tf.keras.Model):
def __init__(self):
super().__init__()
self.flatten = tf.keras.layers.Flatten()
self.rand_weight = tf.constant(tf.random.uniform((20,20)))
self.dense = tf.keras.layers.Dense(units=20, activation=tf.nn.relu)
def call(self, inputs):
x = self.flatten(inputs)
print(x.shape)
x = tf.nn.relu(tf.matmul(x, self.rand_weight) + 1)
print(x.shape)
x = self.dense(x)
print(x.shape)
while tf.norm(x) > 1:
x /= 2
if tf.norm(x) < 0.8:
x *= 10
print(x.shape)
return tf.reduce_sum(x)
net = FancyMLP()
net(X)
(2, 20)
(2, 20)
(2, 20)
(2, 20)
<tf.Tensor: id=229, shape=(), dtype=float32, numpy=22.884422>
class NestMLP(tf.keras.Model):
def __init__(self):
super().__init__()
self.net = tf.keras.Sequential()
self.net.add(tf.keras.layers.Flatten())
self.net.add(tf.keras.layers.Dense(64, activation=tf.nn.relu))
self.net.add(tf.keras.layers.Dense(32, activation=tf.nn.relu))
self.dense = tf.keras.layers.Dense(units=16, activation=tf.nn.relu)
def call(self, inputs):
return self.dense(self.net(inputs))
net = tf.keras.Sequential()
net.add(NestMLP())
net.add(tf.keras.layers.Dense(20))
net.add(FancyMLP())
net(X)
(2, 20)
(2, 20)
(2, 20)
(2, 20)
<tf.Tensor: id=421, shape=(), dtype=float32, numpy=3.6778188>
4.2 模型参数的访问、初始化和共享
net = tf.keras.models.Sequential()
net.add(tf.keras.layers.Flatten())
net.add(tf.keras.layers.Dense(256,activation=tf.nn.relu))
net.add(tf.keras.layers.Dense(10))
X = tf.random.uniform((2,20))
Y = net(X)
Y
<tf.Tensor: id=483, shape=(2, 10), dtype=float32, numpy=
array([[-0.09879929, 0.2812114 , -0.06708632, -0.36011535, 0.2831914 ,
0.26097357, -0.00985743, -0.21124828, -0.1133031 , -0.02720478],
[ 0.07838846, 0.20481616, 0.05093011, -0.25614202, 0.09847108,
0.37754467, -0.07446901, -0.24943468, -0.13108218, 0.24933031]],
dtype=float32)>
4.2.1 access model parameters
net.weights[0]
type(net.weights[0])
<tf.Variable 'sequential_3/dense_10/kernel:0' shape=(20, 256) dtype=float32, numpy=
array([[-0.07033821, 0.04328914, -0.02081853, ..., -0.09415226,
-0.13597511, 0.0630572 ],
[ 0.06236269, -0.12134855, 0.09827602, ..., 0.1120435 ,
0.00854841, 0.05130161],
[-0.0358071 , -0.12622362, 0.05006796, ..., -0.09388144,
0.03304952, 0.11275896],
...,
[ 0.10420921, -0.1144833 , 0.02463526, ..., 0.08800317,
0.0538664 , -0.14500125],
[ 0.10294706, 0.03746136, 0.04467554, ..., -0.05025613,
-0.01331051, 0.05202538],
[-0.0107531 , 0.10216615, 0.04870357, ..., 0.06093717,
-0.08508429, -0.05277579]], dtype=float32)>
tensorflow.python.ops.resource_variable_ops.ResourceVariable
4.2.2 initialize params
class Linear(tf.keras.Model):
def __init__(self):
super().__init__()
self.d1 = tf.keras.layers.Dense(
units=10,
activation=None,
kernel_initializer=tf.zeros_initializer(),
bias_initializer=tf.zeros_initializer()
)
self.d2 = tf.keras.layers.Dense(
units=1,
activation=None,
kernel_initializer=tf.ones_initializer(),
bias_initializer=tf.ones_initializer()
)
def call(self, input):
output = self.d1(input)
output = self.d2(output)
return output
net = Linear()
net(X)
net.get_weights()
<tf.Tensor: id=529, shape=(2, 1), dtype=float32, numpy=
array([[1.],
[1.]], dtype=float32)>
[array([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]], dtype=float32),
array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], dtype=float32),
array([[1.],
[1.],
[1.],
[1.],
[1.],
[1.],
[1.],
[1.],
[1.],
[1.]], dtype=float32),
array([1.], dtype=float32)]
4.2.3 define initializer
def my_init():
return tf.keras.initializers.Ones()
model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Dense(64, kernel_initializer=my_init()))
Y = model(X)
model.weights[0]
<tf.Variable 'sequential_4/dense_14/kernel:0' shape=(20, 64) dtype=float32, numpy=
array([[1., 1., 1., ..., 1., 1., 1.],
[1., 1., 1., ..., 1., 1., 1.],
[1., 1., 1., ..., 1., 1., 1.],
...,
[1., 1., 1., ..., 1., 1., 1.],
[1., 1., 1., ..., 1., 1., 1.],
[1., 1., 1., ..., 1., 1., 1.]], dtype=float32)>
4.4 自定义层
X = tf.random.uniform((2,20))
4.4.1 custom layer without parameters
# 自定义了一个将输入减掉均值后输出的层
class CenteredLayer(tf.keras.layers.Layer):
def __init__(self):
super().__init__()
def call(self, inputs):
return inputs - tf.reduce_mean(inputs)
layer = CenteredLayer()
layer(np.array([1,2,3,4,5]))
<tf.Tensor: id=572, shape=(5,), dtype=int64, numpy=array([-2, -1, 0, 1, 2])>
net = tf.keras.models.Sequential()
net.add(tf.keras.layers.Flatten())
net.add(tf.keras.layers.Dense(20))
net.add(CenteredLayer())
Y = net(X)
Y
<tf.Tensor: id=603, shape=(2, 20), dtype=float32, numpy=
array([[-0.63349915, -0.40265122, 1.2623289 , -1.1872962 , -0.01902312,
-0.8613274 , 1.0604925 , 0.14521688, 1.3011982 , -0.5348058 ,
0.98438555, -0.4622898 , -0.27724132, 1.0449705 , -0.9285989 ,
0.4628783 , 0.52180976, -1.5547564 , -0.4817221 , -0.12175441],
[-0.31887224, 0.17624336, 0.46425617, -0.7229486 , -0.04514135,
-0.38868698, 1.1368203 , 0.03690258, 1.3245962 , -0.12987542,
0.5836614 , -0.4206456 , -0.849625 , 0.22945827, -0.6439618 ,
0.31676728, 0.9253774 , -1.1312481 , -0.39756632, 0.5361735 ]],
dtype=float32)>
Y.shape
tf.reduce_mean(Y)
Y.numpy().mean()
TensorShape([2, 20])
<tf.Tensor: id=605, shape=(), dtype=float32, numpy=-2.9802323e-09>
5.9604646e-09
4.4.2 custom layer with parameters
class myDense(tf.keras.layers.Layer):
def __init__(self, units):
super().__init__()
self.units = units
def build(self, input_shape): # 这里 input_shape 是第一次运行call()时参数inputs的形状
self.w = self.add_weight(name='w',
shape=[input_shape[-1], self.units], initializer=tf.random_normal_initializer())
self.b = self.add_weight(name='b',
shape=[self.units], initializer=tf.zeros_initializer())
def call(self, inputs):
y_pred = tf.matmul(inputs, self.w) + self.b
return y_pred
dense = myDense(3)
dense(X)
dense.get_weights()
<tf.Tensor: id=630, shape=(2, 3), dtype=float32, numpy=
array([[-0.12120296, -0.0113711 , 0.14779961],
[-0.06124324, -0.23015328, 0.15629828]], dtype=float32)>
[array([[ 0.05555374, 0.02348969, 0.0034632 ],
[-0.03911855, -0.07329117, -0.02120967],
[-0.0106332 , 0.01186991, -0.08006126],
[-0.01038213, 0.01291886, -0.09378239],
[ 0.01282662, -0.06754901, 0.09265347],
[ 0.03958872, -0.12498979, 0.09561845],
[-0.00686976, -0.01925179, 0.03493042],
[-0.08048819, 0.07349076, -0.02351638],
[ 0.05533304, -0.07750282, -0.05470523],
[-0.12362194, -0.00875609, 0.01370332],
[ 0.02532404, -0.00088338, 0.02777285],
[ 0.04859733, -0.02423888, -0.02878525],
[ 0.01547945, 0.05682674, 0.08475035],
[-0.05850073, 0.09705635, -0.04129069],
[-0.09561963, 0.04900615, 0.01821384],
[-0.01083881, -0.0033182 , 0.05659602],
[-0.00601833, 0.00671248, 0.03593432],
[-0.02513886, 0.02864583, 0.08744691],
[ 0.04603861, -0.06716666, 0.15384746],
[ 0.07991496, -0.06417349, -0.06345452]], dtype=float32),
array([0., 0., 0.], dtype=float32)]
4.5 读取和存储
4.5.1 load and save NDarray
x = tf.ones(3)
x
<tf.Tensor: id=637, shape=(3,), dtype=float32, numpy=array([1., 1., 1.], dtype=float32)>
np.save('x.npy', x)
x2 = np.load('x.npy')
x2
array([1., 1., 1.], dtype=float32)
y = tf.zeros(4)
np.save('xy.npy',[x, y])
x2, y2 = np.load('xy.npy', allow_pickle=True)
(x2, y2)
(<tf.Tensor: id=641, shape=(3,), dtype=float32, numpy=array([1., 1., 1.], dtype=float32)>,
<tf.Tensor: id=642, shape=(4,), dtype=float32, numpy=array([0., 0., 0., 0.], dtype=float32)>)
mydict = {'x': x, 'y': y}
np.save('mydict.npy', mydict)
mydict2 = np.load('mydict.npy', allow_pickle=True)
mydict2
array({'x': <tf.Tensor: id=643, shape=(3,), dtype=float32, numpy=array([1., 1., 1.], dtype=float32)>, 'y': <tf.Tensor: id=644, shape=(4,), dtype=float32, numpy=array([0., 0., 0., 0.], dtype=float32)>},
dtype=object)
4.5.2 load and save model parameters
X = tf.random.normal((2,20))
X
<tf.Tensor: id=650, shape=(2, 20), dtype=float32, numpy=
array([[-0.70839405, 0.64460796, -1.4520555 , 0.2437724 , 1.2570765 ,
-1.6705031 , -0.31433955, 1.8855785 , -1.133219 , -0.21667722,
-1.4931314 , -1.3752788 , -1.7839235 , -0.14742793, -0.49592182,
1.0912119 , 1.3237706 , -0.15733577, -0.6948644 , -0.09535565],
[-0.29845145, -1.0061126 , 0.55780685, -0.8957967 , -0.20540899,
-1.2849039 , 1.3647797 , -1.1219321 , 0.50872946, 0.48485067,
-0.8191421 , 0.4611657 , -1.9845624 , 0.284558 , -0.60567355,
-0.06300711, 0.6070405 , -0.8394772 , -0.9673258 , -0.37527218]],
dtype=float32)>
class MLP(tf.keras.Model):
def __init__(self):
super().__init__()
self.flatten = tf.keras.layers.Flatten() # Flatten层将除第一维（batch_size）以外的维度展平
self.dense1 = tf.keras.layers.Dense(units=256, activation=tf.nn.relu)
self.dense2 = tf.keras.layers.Dense(units=10)
def call(self, inputs):
x = self.flatten(inputs)
x = self.dense1(x)
output = self.dense2(x)
return output
net = MLP()
Y = net(X)
Y
<tf.Tensor: id=705, shape=(2, 10), dtype=float32, numpy=
array([[-0.7419181 , 0.09955069, 0.7849864 , 0.6915925 , -0.28200203,
-0.1542021 , 0.12158332, -0.96637523, -0.2665258 , -0.98705816],
[-0.2809674 , -0.01123159, 0.3605952 , -0.12920402, 0.68370396,
-0.18064019, 0.09563297, -0.2262938 , -0.28119707, -0.18144041]],
dtype=float32)>
net.save_weights("4.5saved_model.h5")
net2 = MLP()
net2(X)
net2.load_weights("4.5saved_model.h5")
Y2 = net2(X)
Y2 == Y
<tf.Tensor: id=768, shape=(2, 10), dtype=float32, numpy=
array([[-0.03847249, -0.5622585 , -0.2533469 , -0.11625022, 0.02309722,
-0.51970744, -0.939214 , -0.70454395, -0.29292732, -0.5248813 ],
[ 0.18086548, 0.17118181, -0.34515607, -0.518098 , 0.39097375,
-0.34556866, -0.80850315, -0.34327775, 0.3201217 , 0.42236814]],
dtype=float32)>
<tf.Tensor: id=784, shape=(2, 10), dtype=bool, numpy=
array([[ True, True, True, True, True, True, True, True, True,
True],
[ True, True, True, True, True, True, True, True, True,
True]])>
GPU计算
注意：需要tensorflow-gpu
gpus = tf.config.experimental.list_physical_devices(device_type='GPU')
cpus = tf.config.experimental.list_physical_devices(device_type='CPU')
print("可用的GPU：",gpus,"\n可用的CPU：", cpus)
可用的GPU： [PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]
可用的CPU： [PhysicalDevice(name='/physical_device:CPU:0', device_type='CPU')]
check available device
from tensorflow.python.client import device_lib
print(device_lib.list_local_devices())
[name: "/device:CPU:0"
device_type: "CPU"
memory_limit: 268435456
locality {
}
incarnation: 1237627953479486054
, name: "/device:XLA_CPU:0"
device_type: "XLA_CPU"
memory_limit: 17179869184
locality {
}
incarnation: 14330884921366688297
physical_device_desc: "device: XLA_CPU device"
, name: "/device:XLA_GPU:0"
device_type: "XLA_GPU"
memory_limit: 17179869184
locality {
}
incarnation: 156933604364406105
physical_device_desc: "device: XLA_GPU device"
, name: "/device:GPU:0"
device_type: "GPU"
memory_limit: 1108951040
locality {
bus_id: 1
links {
}
}
incarnation: 3744579298438157716
physical_device_desc: "device: 0, name: NVIDIA Tegra X1, pci bus id: 0000:00:00.0, compute capability: 5.3"
]
specify device
with tf.device('GPU:0'):
a = tf.constant([1,2,3],dtype=tf.float32)
b = tf.random.uniform((3,))
print(tf.exp(a + b) * 2)
tf.Tensor([13.942492 20.270329 65.276825], shape=(3,), dtype=float32)