Initial commit
This commit is contained in:
nbwzlyd
commit
75ff1d0516
|
|
@ -0,0 +1,18 @@
|
|||
provinceName,cityName,province_confirmedCount,province_suspectedCount,province_curedCount,province_deadCount,city_confirmedCount,city_suspectedCount,city_curedCount,city_deadCount,updateTime
|
||||
北京市,朝阳区,352,0,56,3,56,0,0,0,2020-02-12 10:35:49.123
|
||||
北京市,海淀区,352,0,56,3,56,0,0,0,2020-02-12 10:35:49.123
|
||||
北京市,西城区,352,0,56,3,43,0,0,0,2020-02-12 10:35:49.123
|
||||
北京市,大兴区,352,0,56,3,37,0,2,0,2020-02-12 10:35:49.123
|
||||
北京市,丰台区,352,0,56,3,32,0,3,0,2020-02-12 10:35:49.123
|
||||
北京市,外地来京人员,352,0,56,3,25,0,2,0,2020-02-12 10:35:49.123
|
||||
北京市,昌平区,352,0,56,3,19,0,0,0,2020-02-12 10:35:49.123
|
||||
北京市,通州区,352,0,56,3,17,0,1,0,2020-02-12 10:35:49.123
|
||||
北京市,房山区,352,0,56,3,14,0,0,0,2020-02-12 10:35:49.123
|
||||
北京市,石景山区,352,0,56,3,13,0,0,0,2020-02-12 10:35:49.123
|
||||
北京市,东城区,352,0,56,3,12,0,0,0,2020-02-12 10:35:49.123
|
||||
北京市,顺义区,352,0,56,3,10,0,0,0,2020-02-12 10:35:49.123
|
||||
北京市,怀柔区,352,0,56,3,7,0,0,0,2020-02-12 10:35:49.123
|
||||
北京市,密云区,352,0,56,3,7,0,0,0,2020-02-12 10:35:49.123
|
||||
北京市,门头沟区,352,0,56,3,3,0,0,0,2020-02-12 10:35:49.123
|
||||
北京市,延庆区,352,0,56,3,1,0,0,0,2020-02-12 10:35:49.123
|
||||
北京市,待明确地区,352,0,56,3,0,0,48,3,2020-02-12 10:35:49.123
|
||||
|
|
|
@ -0,0 +1 @@
|
|||
repo_file.txt
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
#numpy库简单应用
|
||||
import numpy as np
|
||||
import random
|
||||
|
||||
#定义矩阵大小
|
||||
m = 15
|
||||
|
||||
mBtemp = []
|
||||
for x in range(2*m-1):
|
||||
mBtemp.append(random.randint(1, 10))
|
||||
mB = np.matrix(np.array(mBtemp[0:m]))
|
||||
for x in range(1, m):
|
||||
mB = np.insert(mB, x, values=np.array(mBtemp[-1*x:] + mBtemp[0:m-x]), axis = 0)
|
||||
|
||||
vb = []
|
||||
for x in range(m):
|
||||
vb.append(random.randint(1, 10))
|
||||
|
||||
print("B = \n", mB)
|
||||
print("vb = ", vb)
|
||||
|
||||
#invB = np.linalg.inv(mB)
|
||||
|
||||
print("Bx = b, x = ", np.linalg.solve(mB, vb))
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
#简单神经网络测试实例
|
||||
import numpy as np
|
||||
|
||||
def tanh(x): #双曲函数
|
||||
return np.tanh(x)
|
||||
|
||||
def tanh_deriv(x):#更新权重时，需要用到双曲函数的倒数
|
||||
return 1.0 - np.tanh(x)*np.tanh(x)
|
||||
|
||||
def logistic(x):#构建逻辑函数
|
||||
return 1/(1 + np.exp(-x))
|
||||
|
||||
def logistic_derivatic(x): #逻辑函数的倒数
|
||||
return logistic(x)*(1 - logistic(x))
|
||||
|
||||
class NeuralNetwork:
|
||||
def __init__(self,layer,activation='tanh'):
|
||||
'''
|
||||
:param layer:A list containing the number of unit in each layer.
|
||||
Should be at least two values.每层包含的神经元数目
|
||||
:param activation: the activation function to be used.Can be
|
||||
"logistic" or "tanh"
|
||||
'''
|
||||
if activation == 'logistic':
|
||||
self.activation = logistic
|
||||
self.activation_deriv = logistic_derivatic
|
||||
elif activation == 'tanh':
|
||||
self.activation = tanh
|
||||
self.activation_deriv = tanh_deriv
|
||||
|
||||
self.weights = []
|
||||
for i in range(1,len(layer) - 1):#权重的设置
|
||||
self.weights.append((2*np.random.random((layer[i - 1] + 1,layer[i] + 1))-1)*0.25)
|
||||
self.weights.append((2*np.random.random((layer[i] + 1,layer[i+1]))-1)*0.25)
|
||||
'''训练神经网络，通过传入的数据，不断更新权重weights'''
|
||||
def fit(self,X,y,learning_rate=0.2,epochs=10000):
|
||||
'''
|
||||
:param X: 数据集
|
||||
:param y: 数据输出结果，分类标记
|
||||
:param learning_rate: 学习率
|
||||
:param epochs: 随机抽取的数据的训练次数
|
||||
:return:
|
||||
'''
|
||||
X = np.atleast_2d(X) #转化X为np数据类型，试数据类型至少是两维的
|
||||
temp = np.ones([X.shape[0],X.shape[1]+1])
|
||||
temp[:,0:-1] = X
|
||||
X = temp
|
||||
y = np.array(y)
|
||||
|
||||
for k in range(epochs):
|
||||
i = np.random.randint(X.shape[0]) #随机抽取的行
|
||||
a = [X[i]]
|
||||
|
||||
for I in range(len(self.weights)):#完成正向所有的更新
|
||||
a.append(self.activation(np.dot(a[I],self.weights[I])))#dot():对应位相乘后相加
|
||||
error = y[i] - a[-1]
|
||||
deltas = [error * self.activation_deriv(a[-1])]#*self.activation_deriv(a[I])#输出层误差
|
||||
# 反向更新
|
||||
for I in range(len(a) -2,0,-1):
|
||||
deltas.append(deltas[-1].dot(self.weights[I].T)*self.activation_deriv(a[I]))
|
||||
deltas.reverse()
|
||||
for i in range(len(self.weights)):
|
||||
layer = np.atleast_2d(a[i])
|
||||
delta = np.atleast_2d(deltas[i])
|
||||
self.weights[i] += learning_rate*layer.T.dot(delta)
|
||||
|
||||
def predict(self,x):
|
||||
x = np.array(x)
|
||||
temp = np.ones(x.shape[0] + 1)
|
||||
temp[0:-1] = x
|
||||
a = temp
|
||||
for I in range(0,len(self.weights)):
|
||||
a = self.activation(np.dot(a,self.weights[I]))
|
||||
return a #只需要保存最后的值，就是预测出来的值
|
||||
nn = NeuralNetwork([2,2,1], 'tanh')
|
||||
X = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])
|
||||
y = np.array([0, 1, 1, 0])
|
||||
nn.fit(X, y)
|
||||
for i in [[0, 0], [0, 1], [1, 0], [1,1]]:
|
||||
print(i, nn.predict(i))
|
||||
|
||||
#from sklearn.datasets import load_digits #导入数据集
|
||||
#from sklearn.metrics import confusion_matrix,classification_report #对结果的预测的包
|
||||
#from sklearn.preprocessing import LabelBinarizer #把数据转化为二维的数字类型
|
||||
#from sklearn.cross_validation import train_test_split #可以把数据拆分成训练集与数据集
|
||||
|
||||
#digits = load_digits() #把数据改成0到1之间
|
||||
#X = digits.data
|
||||
#y = digits.target
|
||||
#X -= X.min()
|
||||
#X /= X.max()
|
||||
|
||||
#nn = NeuralNetwork([64,100,10],'logistic')
|
||||
#X_train,X_test,y_train,y_test = train_test_split(X,y)
|
||||
#labels_train = LabelBinarizer().fit_transform(y_train)
|
||||
#labels_test = LabelBinarizer().fit_transform(y_test)
|
||||
#print("start fitting")
|
||||
#nn.fit(X_train,labels_train,epochs=3000)
|
||||
#predictions = []
|
||||
#for i in range(X_test.shape[0]):
|
||||
# o = nn.predict(X_test[i])
|
||||
# predictions.append(np.argmax(o))
|
||||
#print(confusion_matrix(y_test,predictions))
|
||||
#print(classification_report(y_test,predictions))
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
import os
|
||||
|
||||
import gym
|
||||
from agit import Agent#之前的是eternatus
|
||||
from gym.spaces import Discrete, Box
|
||||
from ray import tune
|
||||
|
||||
|
||||
class SimpleCorridor(gym.Env):
|
||||
def __init__(self, config):
|
||||
self.end_pos = config['corridor_length']
|
||||
self.cur_pos = 0
|
||||
self.action_space = Discrete(2)
|
||||
self.observation_space = Box(0.0, self.end_pos, shape=(1,))
|
||||
|
||||
def reset(self):
|
||||
self.cur_pos = 0
|
||||
return [self.cur_pos]
|
||||
|
||||
def step(self, action):
|
||||
if action == 0 and self.cur_pos > 0:
|
||||
self.cur_pos -= 1
|
||||
elif action == 1:
|
||||
self.cur_pos += 1
|
||||
done = self.cur_pos >= self.end_pos
|
||||
return [self.cur_pos], 1 if done else 0, done, {}
|
||||
|
||||
|
||||
def main():
|
||||
from datetime import datetime
|
||||
start_time = datetime.utcnow()
|
||||
|
||||
print('Python start time: {} UTC'.format(start_time))
|
||||
|
||||
import tensorflow as tf
|
||||
print('TensorFlow CUDA is available: {}'.format(tf.config.list_physical_devices('GPU')))
|
||||
|
||||
import torch
|
||||
print('pyTorch CUDA is available: {}'.format(torch.cuda.is_available()))
|
||||
|
||||
if 'CLOUD_PROVIDER' in os.environ and os.environ['CLOUD_PROVIDER'] == 'Agit':
|
||||
provider = 'Agit'
|
||||
|
||||
log_dir = '/root/.agit'
|
||||
results_dir = '/root/.agit'
|
||||
else:
|
||||
provider = 'local'
|
||||
|
||||
log_dir = '../temp'
|
||||
results_dir = '../temp'
|
||||
|
||||
# Initialize Ray Cluster
|
||||
#ray_init()
|
||||
|
||||
tune.run(
|
||||
'PPO',
|
||||
queue_trials=True, # Don't use this parameter unless you know what you do.
|
||||
stop={'training_iteration': 10},
|
||||
config={
|
||||
'env': SimpleCorridor,
|
||||
'env_config': {'corridor_length': 5}
|
||||
}
|
||||
)
|
||||
|
||||
with open(os.path.join(results_dir, 'model.pkl'), 'wb') as file:
|
||||
file.write(b'model data')
|
||||
|
||||
complete_time = datetime.utcnow()
|
||||
|
||||
print('Python complete time: {} UTC'.format(complete_time))
|
||||
|
||||
print('Python resource time: {} UTC'.format(complete_time - start_time))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
# The following switch allows the program runs locally and in the Agit environment without modifications.
|
||||
import os
|
||||
|
||||
path = os.path.dirname(__file__)
|
||||
print(path)
|
||||
|
||||
if 'CLOUD_PROVIDER' in os.environ and os.environ['CLOUD_PROVIDER'] == 'Agit':
|
||||
logdir = '/root/.agit'
|
||||
else:
|
||||
logdir = './runs'
|
||||
# setup tensorboard path
|
||||
import tensorflow as tf
|
||||
writer = tf.summary.create_file_writer(logdir)
|
||||
|
||||
''' alternative tensorboards
|
||||
|
||||
# pytorch tensorboard :
|
||||
from torch.utils.tensorboard import SummaryWriter
|
||||
writer = SummaryWriter(log_dir=logdir)
|
||||
|
||||
# tensorboardX :
|
||||
from tensorboardX import SummaryWriter
|
||||
writer = SummaryWriter(logdir=logdir)
|
||||
|
||||
'''
|
||||
|
||||
import numpy as np
|
||||
import time
|
||||
|
||||
# a 5 minutes running example, the realtime tensorboard can be viewed in the training page
|
||||
with writer.as_default():
|
||||
for n_iter in range(360):
|
||||
tf.summary.scalar('Loss/train', np.random.random(), n_iter)
|
||||
tf.summary.scalar('Loss/test', np.random.random(), n_iter)
|
||||
tf.summary.scalar('Accuracy/train', np.random.random(), n_iter)
|
||||
tf.summary.scalar('Accuracy/test', np.random.random(), n_iter)
|
||||
time.sleep(1)
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
import tensorflow as tf
|
||||
import numpy as np
|
||||
import os
|
||||
#这个脚本回到drive的利用率特别高，超出100%
|
||||
"""
|
||||
def load_mnist(path):
|
||||
#加载本地下载好的mnist数据集
|
||||
f = np.load(path)
|
||||
x_train, y_train = f['x_train'], f['y_train']
|
||||
x_test, y_test = f['x_test'], f['y_test']
|
||||
f.close()
|
||||
return (x_train, y_train), (x_test, y_test)
|
||||
|
||||
|
||||
(x_train, y_train), (x_test, y_test) = load_mnist("mnist.npz")
|
||||
"""
|
||||
mnist = tf.keras.datasets.mnist#从xx网站下载mnist到.kera,如果已经有了直接使用
|
||||
|
||||
(x_train, y_train), (x_test, y_test) = mnist.load_data()
|
||||
x_train, x_test = x_train / 255.0, x_test / 255.0 # 将样本从整数转换为浮点数
|
||||
|
||||
# 利用tf.keras.Sequential容器封装网络层，前一层网络的输出默认作为下一层的输入
|
||||
model = tf.keras.models.Sequential([
|
||||
tf.keras.layers.Flatten(input_shape=(28, 28)),
|
||||
tf.keras.layers.Dense(128, activation='relu'), # 创建一层网络，设置输出节点数为128，激活函数类型为Relu
|
||||
tf.keras.layers.Dropout(0.2), # 在训练中每次更新时， 将输入单元的按比率随机设置为 0， 这有助于防止过拟合
|
||||
tf.keras.layers.Dense(10, activation='softmax')]) # Dense层就是所谓的全连接神经网络层
|
||||
|
||||
model.summary()#显示模型的结构
|
||||
|
||||
# 为训练选择优化器和损失函数：
|
||||
model.compile(optimizer='adam',
|
||||
loss='sparse_categorical_crossentropy',
|
||||
metrics=['accuracy'])
|
||||
if 'CLOUD_PROVIDER' in os.environ and os.environ['CLOUD_PROVIDER'] == 'Agit':
|
||||
log_dir = os.path.join('/root/.agit/logs') # this is the storage path in the Agit environment
|
||||
else:
|
||||
log_dir = os.path.join("logs") # this is the path when the program runs in other environments
|
||||
#log_dir = os.path.join("logs")
|
||||
# print(log_dir)
|
||||
if not os.path.exists(log_dir):
|
||||
os.mkdir(log_dir)
|
||||
# 定义TensorBoard对象.histogram_freq 如果设置为0，则不会计算直方图。
|
||||
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)
|
||||
|
||||
# TensorBoard对象作为回调传给model.fit方法
|
||||
model.fit(x_train, y_train, epochs=8, validation_data=(x_test, y_test), callbacks=[tensorboard_callback])
|
||||
|
||||
model.save_weights(log_dir + '/weight/my_weights', save_format='tf') # 保存模型*****直接引用对应的路径参数
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
import os
|
||||
#测试单个gpu，没有worker的资源脚本
|
||||
import gym
|
||||
import ray
|
||||
from gym.spaces import Discrete, Box
|
||||
from ray import tune
|
||||
|
||||
|
||||
class SimpleCorridor(gym.Env):
|
||||
def __init__(self, config):
|
||||
self.end_pos = config['corridor_length']
|
||||
self.cur_pos = 0
|
||||
self.action_space = Discrete(2)
|
||||
self.observation_space = Box(0.0, self.end_pos, shape=(1,))
|
||||
|
||||
def reset(self):
|
||||
self.cur_pos = 0
|
||||
return [self.cur_pos]
|
||||
|
||||
def step(self, action):
|
||||
if action == 0 and self.cur_pos > 0:
|
||||
self.cur_pos -= 1
|
||||
elif action == 1:
|
||||
self.cur_pos += 1
|
||||
done = self.cur_pos >= self.end_pos
|
||||
return [self.cur_pos], 1 if done else 0, done, {}
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
from datetime import datetime
|
||||
|
||||
start_time = datetime.utcnow()
|
||||
|
||||
print('Python start time: {} UTC'.format(start_time))
|
||||
|
||||
if 'CLOUD_PROVIDER' in os.environ and os.environ['CLOUD_PROVIDER'] == 'Agit':
|
||||
from agit import ray_init
|
||||
|
||||
ray_init()
|
||||
else:
|
||||
ray.init()
|
||||
|
||||
print('Ray Cluster Resources: {}'.format(ray.cluster_resources()))
|
||||
|
||||
import tensorflow as tf
|
||||
|
||||
print('TensorFlow CUDA is available: {}'.format(tf.config.list_physical_devices('GPU')))
|
||||
|
||||
import torch
|
||||
|
||||
print('pyTorch CUDA is available: {}'.format(torch.cuda.is_available()))
|
||||
|
||||
tune.run(
|
||||
'PPO',
|
||||
queue_trials=True, # Don't use this parameter unless you know what you do.
|
||||
stop={'training_iteration': 10},
|
||||
config={
|
||||
'env': SimpleCorridor,
|
||||
'env_config': {'corridor_length': 5},
|
||||
'num_gpus': 1
|
||||
}
|
||||
)
|
||||
|
||||
complete_time = datetime.utcnow()
|
||||
|
||||
print('Python complete time: {} UTC'.format(complete_time))
|
||||
|
||||
print('Python resource time: {} UTC'.format(complete_time - start_time))
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
import os
|
||||
#测试gpu，有worker的资源脚本
|
||||
import gym
|
||||
import ray
|
||||
from gym.spaces import Discrete, Box
|
||||
from ray import tune
|
||||
|
||||
|
||||
class SimpleCorridor(gym.Env):
|
||||
def __init__(self, config):
|
||||
self.end_pos = config['corridor_length']
|
||||
self.cur_pos = 0
|
||||
self.action_space = Discrete(2)
|
||||
self.observation_space = Box(0.0, self.end_pos, shape=(1,))
|
||||
|
||||
def reset(self):
|
||||
self.cur_pos = 0
|
||||
return [self.cur_pos]
|
||||
|
||||
def step(self, action):
|
||||
if action == 0 and self.cur_pos > 0:
|
||||
self.cur_pos -= 1
|
||||
elif action == 1:
|
||||
self.cur_pos += 1
|
||||
done = self.cur_pos >= self.end_pos
|
||||
return [self.cur_pos], 1 if done else 0, done, {}
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
from datetime import datetime
|
||||
|
||||
start_time = datetime.utcnow()
|
||||
|
||||
print('Python start time: {} UTC'.format(start_time))
|
||||
|
||||
if 'CLOUD_PROVIDER' in os.environ and os.environ['CLOUD_PROVIDER'] == 'Agit':
|
||||
from agit import ray_init
|
||||
|
||||
ray_init()
|
||||
else:
|
||||
ray.init()
|
||||
|
||||
print('Ray Cluster Resources: {}'.format(ray.cluster_resources()))
|
||||
|
||||
import tensorflow as tf
|
||||
|
||||
print('TensorFlow CUDA is available: {}'.format(tf.config.list_physical_devices('GPU')))
|
||||
|
||||
import torch
|
||||
|
||||
print('pyTorch CUDA is available: {}'.format(torch.cuda.is_available()))
|
||||
|
||||
tune.run(
|
||||
'PPO',
|
||||
queue_trials=True, # Don't use this parameter unless you know what you do.
|
||||
stop={'training_iteration': 10},
|
||||
config={
|
||||
'env': SimpleCorridor,
|
||||
'env_config': {'corridor_length': 5},
|
||||
'num_gpus': 1,
|
||||
'num_gpus_per_worker': 1,
|
||||
},
|
||||
)
|
||||
|
||||
complete_time = datetime.utcnow()
|
||||
|
||||
print('Python complete time: {} UTC'.format(complete_time))
|
||||
|
||||
print('Python resource time: {} UTC'.format(complete_time - start_time))
|
||||
|
|
@ -0,0 +1,100 @@
|
|||
import torch as t
|
||||
import torchvision as tv
|
||||
import torchvision.transforms as transforms
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
#错误的还未学习
|
||||
########## 超参数设置 ##########
|
||||
epochs = 200
|
||||
learning_rate = 0.001
|
||||
batch_size = 256
|
||||
gpu_ids = [0, 1, 2]
|
||||
|
||||
|
||||
########## 一、数据加载与预处理 ##########
|
||||
transform = transforms.Compose([
|
||||
transforms.ToTensor(),
|
||||
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
|
||||
])
|
||||
trainset = tv.datasets.CIFAR10(root='./data/', train=True, download=True, transform=transform)
|
||||
testset = tv.datasets.CIFAR10(root='./data/', train=False, download=True, transform=transform)
|
||||
trainloader = t.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=4)
|
||||
testloader = t.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=4)
|
||||
|
||||
|
||||
########## 二、定义神经网络 ##########
|
||||
class LeNet(nn.Module):
|
||||
def __init__(self):
|
||||
super(LeNet, self).__init__()
|
||||
# (in_channels, out_channels, kernel_size, stride, padding)
|
||||
self.conv1 = nn.Conv2d(3, 6, 5, 1, 0)
|
||||
self.conv2 = nn.Conv2d(6, 16, 5, 1, 0)
|
||||
self.fc1 = nn.Linear(16 * 5 * 5, 120)
|
||||
self.fc2 = nn.Linear(120, 84)
|
||||
self.fc3 = nn.Linear(84, 10)
|
||||
|
||||
def forward(self, x):
|
||||
x = F.max_pool2d(F.relu(self.conv1(x)), 2, 2)
|
||||
x = F.max_pool2d(F.relu(self.conv2(x)), 2, 2)
|
||||
x = x.view(x.size()[0], -1)
|
||||
x = F.relu(self.fc1(x))
|
||||
x = F.relu(self.fc2(x))
|
||||
x = self.fc3(x)
|
||||
return x
|
||||
|
||||
|
||||
########## 三、训练神经网络 ##########
|
||||
# 神经网络实例化
|
||||
net = LeNet()
|
||||
if (len(gpu_ids) == 0) or (not t.cuda.is_available()):
|
||||
# gpu_ids列表为空, 或者没GPU可用，则设置为GPU模式。
|
||||
device = t.device("cpu")
|
||||
print("Train Mode : CPU")
|
||||
elif t.cuda.is_available() and len(gpu_ids) > 1:
|
||||
# gpu_ids列表大于1，表明想用多个GPU训练
|
||||
device = t.device("cuda:0")
|
||||
net = nn.DataParallel(net, device_ids=gpu_ids)
|
||||
print("Train Mode : Multi GPU;", gpu_ids)
|
||||
else:
|
||||
# gpu_ids列表等于1，表明想用一个GPU训练
|
||||
device = t.device("cuda:" + str(gpu_ids[0]) if t.cuda.is_available() else "cpu")
|
||||
print("Train Mode : One GPU;", device)
|
||||
net = net.to(device)
|
||||
print("\n", "##" * 10, " NetWork ", "##" * 10, "\n", net, "\n", "##" * 26, "\n")
|
||||
# 定义损失函数和优化器
|
||||
criterion = nn.CrossEntropyLoss()
|
||||
optimizer = t.optim.SGD(net.parameters(), lr=learning_rate, momentum=0.9)
|
||||
# 开始训练
|
||||
for epoch in range(epochs):
|
||||
running_loss = 0.0
|
||||
for i, (inputs, labels) in enumerate(trainloader):
|
||||
inputs, labels = inputs.to(device), labels.to(device)
|
||||
optimizer.zero_grad()
|
||||
outputs = net(inputs)
|
||||
loss = criterion(outputs, labels)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
running_loss += loss.item()
|
||||
print("Epoch%03d: Training_loss = %.5f" % (epoch + 1, running_loss))
|
||||
|
||||
# 保存模型并进行验证
|
||||
if (epoch + 1) % 10 == 0:
|
||||
if len(gpu_ids) > 1:
|
||||
t.save(net.module.state_dict(), "Cifar10_LeNet_Epoch" + str(epoch + 1) + ".pth")
|
||||
else:
|
||||
t.save(net.state_dict(), "Cifar10_LeNet_Epoch" + str(epoch + 1) + ".pth")
|
||||
correct = 0
|
||||
all = 0
|
||||
with t.no_grad():
|
||||
for (inputs, labels) in testloader:
|
||||
inputs, labels = inputs.to(device), labels.to(device)
|
||||
outputs = net(inputs)
|
||||
_, predicted = t.max(outputs, 1)
|
||||
all += inputs.size()[0]
|
||||
correct += (predicted == labels).sum().item()
|
||||
print("###" * 15)
|
||||
print("Epoch%03d: TestSet_Accuracy = %.3f" % (epoch + 1, correct / all))
|
||||
print("###" * 15)
|
||||
|
||||
print("Train Done!")
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
from __future__ import absolute_import, division, print_function, unicode_literals
|
||||
import tensorflow as tf
|
||||
|
||||
mnist = tf.keras.datasets.mnist
|
||||
#需要从 https://storage.googleapis.com/tensorflow/tf-keras-datasets/ 下载，执行过程非常缓慢，或者报证书错误可以直接从浏览器下载 https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz
|
||||
#并保存到 ~/.kreas/datasets/ 目录下（c盘根目录）即可
|
||||
|
||||
(x_train, y_train), (x_test, y_test) = mnist.load_data()
|
||||
x_train, x_test = x_train / 255.0, x_test / 255.0
|
||||
|
||||
model = tf.keras.models.Sequential([
|
||||
tf.keras.layers.Flatten(input_shape=(28, 28)),
|
||||
tf.keras.layers.Dense(128, activation='relu'),
|
||||
tf.keras.layers.Dropout(0.2),
|
||||
tf.keras.layers.Dense(10, activation='softmax')
|
||||
])
|
||||
|
||||
model.compile(optimizer='adam',
|
||||
loss='sparse_categorical_crossentropy',
|
||||
metrics=['accuracy'])
|
||||
|
||||
model.fit(x_train, y_train, epochs=5)
|
||||
model.evaluate(x_test, y_test)
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
#调用 TensorFlow
|
||||
from __future__ import absolute_import, division, print_function, unicode_literals
|
||||
|
||||
import tensorflow as tf
|
||||
|
||||
from tensorflow.keras.layers import Dense, Flatten, Conv2D
|
||||
from tensorflow.keras import Model
|
||||
|
||||
#载入并准备好 MNIST 数据集：
|
||||
mnist = tf.keras.datasets.mnist
|
||||
|
||||
(x_train, y_train), (x_test, y_test) = mnist.load_data()
|
||||
x_train, x_test = x_train / 255.0, x_test / 255.0
|
||||
|
||||
# Add a channels dimension
|
||||
x_train = x_train[..., tf.newaxis]
|
||||
x_test = x_test[..., tf.newaxis]
|
||||
|
||||
|
||||
#使用 tf.data 来将数据集切分为 batch 以及混淆数据集：
|
||||
train_ds = tf.data.Dataset.from_tensor_slices(
|
||||
(x_train, y_train)).shuffle(10000).batch(32)
|
||||
test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(32)
|
||||
|
||||
#使用 Keras 模型子类化（model subclassing） API 构建 tf.keras 模型：
|
||||
class MyModel(Model):
|
||||
def __init__(self):
|
||||
super(MyModel, self).__init__()
|
||||
self.conv1 = Conv2D(32, 3, activation='relu')
|
||||
self.flatten = Flatten()
|
||||
self.d1 = Dense(128, activation='relu')
|
||||
self.d2 = Dense(10, activation='softmax')
|
||||
|
||||
def call(self, x):
|
||||
x = self.conv1(x)
|
||||
x = self.flatten(x)
|
||||
x = self.d1(x)
|
||||
return self.d2(x)
|
||||
|
||||
|
||||
model = MyModel()
|
||||
|
||||
#为训练选择优化器与损失函数：
|
||||
loss_object = tf.keras.losses.SparseCategoricalCrossentropy()
|
||||
|
||||
optimizer = tf.keras.optimizers.Adam()
|
||||
|
||||
#选择衡量指标来度量模型的损失值（loss）和准确率（accuracy）。这些指标在 epoch 上累积值，然后打印出整体结果
|
||||
train_loss = tf.keras.metrics.Mean(name='train_loss')
|
||||
train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(
|
||||
name='train_accuracy')
|
||||
|
||||
test_loss = tf.keras.metrics.Mean(name='test_loss')
|
||||
test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(
|
||||
name='test_accuracy')
|
||||
|
||||
#使用 tf.GradientTape 来训练模型
|
||||
@tf.function
|
||||
def train_step(images, labels):
|
||||
with tf.GradientTape() as tape:
|
||||
predictions = model(images)
|
||||
loss = loss_object(labels, predictions)
|
||||
gradients = tape.gradient(loss, model.trainable_variables)
|
||||
optimizer.apply_gradients(zip(gradients, model.trainable_variables))
|
||||
|
||||
train_loss(loss)
|
||||
train_accuracy(labels, predictions)
|
||||
|
||||
#测试模型：
|
||||
@tf.function
|
||||
def test_step(images, labels):
|
||||
predictions = model(images)
|
||||
t_loss = loss_object(labels, predictions)
|
||||
|
||||
test_loss(t_loss)
|
||||
test_accuracy(labels, predictions)
|
||||
|
||||
|
||||
EPOCHS = 5
|
||||
|
||||
for epoch in range(EPOCHS):
|
||||
for images, labels in train_ds:
|
||||
train_step(images, labels)
|
||||
|
||||
for test_images, test_labels in test_ds:
|
||||
test_step(test_images, test_labels)
|
||||
|
||||
template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}'
|
||||
print(template.format(epoch + 1,
|
||||
train_loss.result(),
|
||||
train_accuracy.result() * 100,
|
||||
test_loss.result(),
|
||||
test_accuracy.result() * 100))
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
import os
|
||||
|
||||
'''
|
||||
This hacks the python built-in function "open" which add some pre-processing for file operations
|
||||
to allows the program runs locally and in the Agit environment without modifications.
|
||||
'''
|
||||
if 'CLOUD_PROVIDER' in os.environ and os.environ['CLOUD_PROVIDER'] == 'Agit':
|
||||
from agit import open # override the open function
|
||||
dataset_path = 'agit://' # data path in the Agit cloud environment
|
||||
else:
|
||||
dataset_path = './dataset/' # data path for local running
|
||||
|
||||
'''
|
||||
Agit Datasets only allow read-only mode, the default mode "r" (open for reading text, synonym of "rt")
|
||||
and "rb " (open for reading binary) are available.
|
||||
'''
|
||||
with open(dataset_path + 'datafile.txt', mode='rb', encoding=None) as file:
|
||||
print(file.read())
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
import os
|
||||
|
||||
import gym
|
||||
import ray
|
||||
from gym.spaces import Discrete, Box
|
||||
from ray import tune
|
||||
|
||||
|
||||
class SimpleCorridor(gym.Env):
|
||||
def __init__(self, config):
|
||||
self.end_pos = config['corridor_length']
|
||||
self.cur_pos = 0
|
||||
self.action_space = Discrete(2)
|
||||
self.observation_space = Box(0.0, self.end_pos, shape=(1,))
|
||||
|
||||
def reset(self):
|
||||
self.cur_pos = 0
|
||||
return [self.cur_pos]
|
||||
|
||||
def step(self, action):
|
||||
if action == 0 and self.cur_pos > 0:
|
||||
self.cur_pos -= 1
|
||||
elif action == 1:
|
||||
self.cur_pos += 1
|
||||
done = self.cur_pos >= self.end_pos
|
||||
return [self.cur_pos], 1 if done else 0, done, {}
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
from datetime import datetime
|
||||
|
||||
start_time = datetime.utcnow()
|
||||
|
||||
print('Python start time: {} UTC'.format(start_time))
|
||||
|
||||
if 'CLOUD_PROVIDER' in os.environ and os.environ['CLOUD_PROVIDER'] == 'Agit':
|
||||
from agit import ray_init
|
||||
|
||||
ray_init()
|
||||
|
||||
from agit import open
|
||||
|
||||
dataset_path = 'agit://'
|
||||
else:
|
||||
ray.init()
|
||||
|
||||
dataset_path = './'
|
||||
|
||||
print('Ray Cluster Resources: {}'.format(ray.cluster_resources()))
|
||||
|
||||
import tensorflow as tf
|
||||
|
||||
print('TensorFlow CUDA is available: {}'.format(tf.config.list_physical_devices('GPU')))
|
||||
|
||||
import torch
|
||||
|
||||
print('pyTorch CUDA is available: {}'.format(torch.cuda.is_available()))
|
||||
|
||||
with open(dataset_path + 'expert_data.csv', 'rb') as file:
|
||||
raw_data = file.read()
|
||||
|
||||
print(raw_data)
|
||||
|
||||
tune.run(
|
||||
'PPO',
|
||||
queue_trials=True, # Don't use this parameter unless you know what you do.
|
||||
stop={'training_iteration': 10},
|
||||
config={
|
||||
'env': SimpleCorridor,
|
||||
'env_config': {'corridor_length': 5},
|
||||
'num_gpus': 1
|
||||
}
|
||||
)
|
||||
|
||||
complete_time = datetime.utcnow()
|
||||
|
||||
print('Python complete time: {} UTC'.format(complete_time))
|
||||
|
||||
print('Python resource time: {} UTC'.format(complete_time - start_time))
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
from pathlib import Path # pathlib is recommended
|
||||
|
||||
#repofile = Path('../repo_file.txt') # 读取上一级目录文件
|
||||
repofile = Path('./repo_file.txt') # 读取上一级目录文件
|
||||
with repofile.open() as f:
|
||||
print(f.readline())
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
1.ray脚本用户输入的路径不会影响agit的输出路径
|
||||
2.tensorflow2.0脚本用户输入的路径：例如当前文件夹“./”，目前不会再训练结果中生成，以为agit系统回默认输出保存再/root/.Agit下的，目前阶段paas端再修改SDK，方便使用update
|
||||
方法上传
|
||||
3.没有在指定路径下生成的文件夹等，例如模型，是否会销毁（垃圾回收）
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
import os
|
||||
from pathlib import Path # pathlib is recommended
|
||||
|
||||
'''
|
||||
The following switch allows the program runs locally and in the Agit environment without modifications.
|
||||
'''
|
||||
if 'CLOUD_PROVIDER' in os.environ and os.environ['CLOUD_PROVIDER'] == 'Agit':
|
||||
result_path = Path('/root/.agit') # this is the storage path in the Agit environment
|
||||
else:
|
||||
result_path = Path('.') # this is the path when the program runs in other environments
|
||||
|
||||
outputFile = result_path / 'output.txt'
|
||||
outputFile.write_text('Hello Agit storage!')
|
||||
Loading…
Reference in New Issue