Skip to content

Commit 440a18d

Browse files
yunjeyyunjey
committed
add dqn
1 parent cd4f2d5 commit 440a18d

File tree

3 files changed

+637
-0
lines changed

3 files changed

+637
-0
lines changed

‎tutorials/11 - Deep Q Network/ReplayMemory.ipynb‎

Lines changed: 359 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,359 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"metadata": {},
6+
"source": [
7+
"# PyTorch DQN Implemenation\n",
8+
"\n",
9+
"<br/>"
10+
]
11+
},
12+
{
13+
"cell_type": "code",
14+
"execution_count": 1,
15+
"metadata": {
16+
"collapsed": false
17+
},
18+
"outputs": [],
19+
"source": [
20+
"%matplotlib inline\n",
21+
"\n",
22+
"import torch\n",
23+
"import torch.nn as nn\n",
24+
"import gym\n",
25+
"import random\n",
26+
"import numpy as np\n",
27+
"import torchvision.transforms as transforms\n",
28+
"import matplotlib.pyplot as plt\n",
29+
"from torch.autograd import Variable\n",
30+
"from collections import deque, namedtuple"
31+
]
32+
},
33+
{
34+
"cell_type": "code",
35+
"execution_count": 2,
36+
"metadata": {
37+
"collapsed": false
38+
},
39+
"outputs": [
40+
{
41+
"name": "stderr",
42+
"output_type": "stream",
43+
"text": [
44+
"[2017-03-09 21:31:48,174] Making new env: CartPole-v0\n"
45+
]
46+
}
47+
],
48+
"source": [
49+
"env = gym.envs.make(\"CartPole-v0\")"
50+
]
51+
},
52+
{
53+
"cell_type": "code",
54+
"execution_count": 3,
55+
"metadata": {
56+
"collapsed": true
57+
},
58+
"outputs": [],
59+
"source": [
60+
"class Net(nn.Module):\n",
61+
" def __init__(self):\n",
62+
" super(Net, self).__init__()\n",
63+
" self.fc1 = nn.Linear(4, 128)\n",
64+
" self.tanh = nn.Tanh()\n",
65+
" self.fc2 = nn.Linear(128, 2)\n",
66+
" self.init_weights()\n",
67+
" \n",
68+
" def init_weights(self):\n",
69+
" self.fc1.weight.data.uniform_(-0.1, 0.1)\n",
70+
" self.fc2.weight.data.uniform_(-0.1, 0.1)\n",
71+
" \n",
72+
" def forward(self, x):\n",
73+
" out = self.fc1(x)\n",
74+
" out = self.tanh(out)\n",
75+
" out = self.fc2(out)\n",
76+
" return out"
77+
]
78+
},
79+
{
80+
"cell_type": "code",
81+
"execution_count": 4,
82+
"metadata": {
83+
"collapsed": false
84+
},
85+
"outputs": [],
86+
"source": [
87+
"def make_epsilon_greedy_policy(network, epsilon, nA):\n",
88+
" def policy(state):\n",
89+
" sample = random.random()\n",
90+
" if sample < (1-epsilon) + (epsilon/nA):\n",
91+
" q_values = network(state.view(1, -1))\n",
92+
" action = q_values.data.max(1)[1][0, 0]\n",
93+
" else:\n",
94+
" action = random.randrange(nA)\n",
95+
" return action\n",
96+
" return policy"
97+
]
98+
},
99+
{
100+
"cell_type": "code",
101+
"execution_count": 5,
102+
"metadata": {
103+
"collapsed": true
104+
},
105+
"outputs": [],
106+
"source": [
107+
"class ReplayMemory(object):\n",
108+
" \n",
109+
" def __init__(self, capacity):\n",
110+
" self.memory = deque()\n",
111+
" self.capacity = capacity\n",
112+
" \n",
113+
" def push(self, transition):\n",
114+
" if len(self.memory) > self.capacity:\n",
115+
" self.memory.popleft()\n",
116+
" self.memory.append(transition)\n",
117+
" \n",
118+
" def sample(self, batch_size):\n",
119+
" return random.sample(self.memory, batch_size)\n",
120+
" \n",
121+
" def __len__(self):\n",
122+
" return len(self.memory)"
123+
]
124+
},
125+
{
126+
"cell_type": "code",
127+
"execution_count": 6,
128+
"metadata": {
129+
"collapsed": true
130+
},
131+
"outputs": [],
132+
"source": [
133+
"def to_tensor(ndarray, volatile=False):\n",
134+
" return Variable(torch.from_numpy(ndarray), volatile=volatile).float()"
135+
]
136+
},
137+
{
138+
"cell_type": "code",
139+
"execution_count": 7,
140+
"metadata": {
141+
"collapsed": false
142+
},
143+
"outputs": [],
144+
"source": [
145+
"def deep_q_learning(num_episodes=10, batch_size=100, \n",
146+
" discount_factor=0.95, epsilon=0.1, epsilon_decay=0.95):\n",
147+
"\n",
148+
" # Q-Network and memory \n",
149+
" net = Net()\n",
150+
" memory = ReplayMemory(10000)\n",
151+
" \n",
152+
" # Loss and Optimizer\n",
153+
" criterion = nn.MSELoss()\n",
154+
" optimizer = torch.optim.Adam(net.parameters(), lr=0.001)\n",
155+
" \n",
156+
" for i_episode in range(num_episodes):\n",
157+
" \n",
158+
" # Set policy (TODO: decaying epsilon)\n",
159+
" #if (i_episode+1) % 100 == 0:\n",
160+
" # epsilon *= 0.9\n",
161+
" \n",
162+
" policy = make_epsilon_greedy_policy(\n",
163+
" net, epsilon, env.action_space.n)\n",
164+
" \n",
165+
" # Start an episode\n",
166+
" state = env.reset()\n",
167+
" \n",
168+
" for t in range(10000):\n",
169+
" \n",
170+
" # Sample action from epsilon greed policy\n",
171+
" action = policy(to_tensor(state)) \n",
172+
" next_state, reward, done, _ = env.step(action)\n",
173+
" \n",
174+
" \n",
175+
" # Restore transition in memory\n",
176+
" memory.push([state, action, reward, next_state])\n",
177+
" \n",
178+
" \n",
179+
" if len(memory) >= batch_size:\n",
180+
" # Sample mini-batch transitions from memory\n",
181+
" batch = memory.sample(batch_size)\n",
182+
" state_batch = np.vstack([trans[0] for trans in batch])\n",
183+
" action_batch =np.vstack([trans[1] for trans in batch]) \n",
184+
" reward_batch = np.vstack([trans[2] for trans in batch])\n",
185+
" next_state_batch = np.vstack([trans[3] for trans in batch])\n",
186+
" \n",
187+
" # Forward + Backward + Opimize\n",
188+
" net.zero_grad()\n",
189+
" q_values = net(to_tensor(state_batch))\n",
190+
" next_q_values = net(to_tensor(next_state_batch, volatile=True))\n",
191+
" next_q_values.volatile = False\n",
192+
" \n",
193+
" td_target = to_tensor(reward_batch) + discount_factor * (next_q_values).max(1)[0]\n",
194+
" loss = criterion(q_values.gather(1, \n",
195+
" to_tensor(action_batch).long().view(-1, 1)), td_target)\n",
196+
" loss.backward()\n",
197+
" optimizer.step()\n",
198+
" \n",
199+
" if done:\n",
200+
" break\n",
201+
" \n",
202+
" state = next_state\n",
203+
" \n",
204+
" if len(memory) >= batch_size and (i_episode+1) % 10 == 0:\n",
205+
" print ('episode: %d, time: %d, loss: %.4f' %(i_episode, t, loss.data[0]))\n",
206+
" "
207+
]
208+
},
209+
{
210+
"cell_type": "code",
211+
"execution_count": 8,
212+
"metadata": {
213+
"collapsed": false
214+
},
215+
"outputs": [
216+
{
217+
"name": "stdout",
218+
"output_type": "stream",
219+
"text": [
220+
"episode: 9, time: 9, loss: 0.9945\n",
221+
"episode: 19, time: 9, loss: 1.8221\n",
222+
"episode: 29, time: 9, loss: 4.3124\n",
223+
"episode: 39, time: 8, loss: 6.9764\n",
224+
"episode: 49, time: 9, loss: 6.8300\n",
225+
"episode: 59, time: 8, loss: 5.5186\n",
226+
"episode: 69, time: 9, loss: 4.1160\n",
227+
"episode: 79, time: 9, loss: 2.4802\n",
228+
"episode: 89, time: 13, loss: 0.7890\n",
229+
"episode: 99, time: 10, loss: 0.2805\n",
230+
"episode: 109, time: 12, loss: 0.1323\n",
231+
"episode: 119, time: 13, loss: 0.0519\n",
232+
"episode: 129, time: 18, loss: 0.0176\n",
233+
"episode: 139, time: 22, loss: 0.0067\n",
234+
"episode: 149, time: 17, loss: 0.0114\n",
235+
"episode: 159, time: 26, loss: 0.0017\n",
236+
"episode: 169, time: 23, loss: 0.0018\n",
237+
"episode: 179, time: 21, loss: 0.0023\n",
238+
"episode: 189, time: 11, loss: 0.0024\n",
239+
"episode: 199, time: 7, loss: 0.0040\n",
240+
"episode: 209, time: 8, loss: 0.0030\n",
241+
"episode: 219, time: 7, loss: 0.0070\n",
242+
"episode: 229, time: 9, loss: 0.0031\n",
243+
"episode: 239, time: 9, loss: 0.0029\n",
244+
"episode: 249, time: 8, loss: 0.0046\n",
245+
"episode: 259, time: 8, loss: 0.0009\n",
246+
"episode: 269, time: 10, loss: 0.0020\n",
247+
"episode: 279, time: 9, loss: 0.0025\n",
248+
"episode: 289, time: 8, loss: 0.0015\n",
249+
"episode: 299, time: 10, loss: 0.0009\n",
250+
"episode: 309, time: 8, loss: 0.0012\n",
251+
"episode: 319, time: 8, loss: 0.0034\n",
252+
"episode: 329, time: 8, loss: 0.0008\n",
253+
"episode: 339, time: 9, loss: 0.0021\n",
254+
"episode: 349, time: 8, loss: 0.0018\n",
255+
"episode: 359, time: 9, loss: 0.0017\n",
256+
"episode: 369, time: 9, loss: 0.0006\n",
257+
"episode: 379, time: 9, loss: 0.0023\n",
258+
"episode: 389, time: 10, loss: 0.0017\n",
259+
"episode: 399, time: 8, loss: 0.0018\n",
260+
"episode: 409, time: 8, loss: 0.0023\n",
261+
"episode: 419, time: 9, loss: 0.0020\n",
262+
"episode: 429, time: 9, loss: 0.0006\n",
263+
"episode: 439, time: 10, loss: 0.0006\n",
264+
"episode: 449, time: 10, loss: 0.0025\n",
265+
"episode: 459, time: 9, loss: 0.0013\n",
266+
"episode: 469, time: 8, loss: 0.0011\n",
267+
"episode: 479, time: 8, loss: 0.0005\n",
268+
"episode: 489, time: 8, loss: 0.0004\n",
269+
"episode: 499, time: 7, loss: 0.0017\n",
270+
"episode: 509, time: 7, loss: 0.0004\n",
271+
"episode: 519, time: 10, loss: 0.0008\n",
272+
"episode: 529, time: 11, loss: 0.0006\n",
273+
"episode: 539, time: 9, loss: 0.0010\n",
274+
"episode: 549, time: 8, loss: 0.0006\n",
275+
"episode: 559, time: 8, loss: 0.0012\n",
276+
"episode: 569, time: 9, loss: 0.0011\n",
277+
"episode: 579, time: 8, loss: 0.0010\n",
278+
"episode: 589, time: 8, loss: 0.0008\n",
279+
"episode: 599, time: 10, loss: 0.0010\n",
280+
"episode: 609, time: 8, loss: 0.0005\n",
281+
"episode: 619, time: 9, loss: 0.0004\n",
282+
"episode: 629, time: 8, loss: 0.0007\n",
283+
"episode: 639, time: 10, loss: 0.0014\n",
284+
"episode: 649, time: 10, loss: 0.0004\n",
285+
"episode: 659, time: 9, loss: 0.0008\n",
286+
"episode: 669, time: 8, loss: 0.0005\n",
287+
"episode: 679, time: 8, loss: 0.0003\n",
288+
"episode: 689, time: 9, loss: 0.0009\n",
289+
"episode: 699, time: 8, loss: 0.0004\n",
290+
"episode: 709, time: 8, loss: 0.0013\n",
291+
"episode: 719, time: 8, loss: 0.0006\n",
292+
"episode: 729, time: 7, loss: 0.0021\n",
293+
"episode: 739, time: 9, loss: 0.0023\n",
294+
"episode: 749, time: 9, loss: 0.0039\n",
295+
"episode: 759, time: 8, loss: 0.0030\n",
296+
"episode: 769, time: 9, loss: 0.0016\n",
297+
"episode: 779, time: 7, loss: 0.0041\n",
298+
"episode: 789, time: 8, loss: 0.0050\n",
299+
"episode: 799, time: 8, loss: 0.0041\n",
300+
"episode: 809, time: 11, loss: 0.0053\n",
301+
"episode: 819, time: 7, loss: 0.0018\n",
302+
"episode: 829, time: 9, loss: 0.0019\n",
303+
"episode: 839, time: 11, loss: 0.0017\n",
304+
"episode: 849, time: 8, loss: 0.0029\n",
305+
"episode: 859, time: 9, loss: 0.0012\n",
306+
"episode: 869, time: 9, loss: 0.0036\n",
307+
"episode: 879, time: 7, loss: 0.0017\n",
308+
"episode: 889, time: 9, loss: 0.0016\n",
309+
"episode: 899, time: 10, loss: 0.0023\n",
310+
"episode: 909, time: 8, loss: 0.0032\n",
311+
"episode: 919, time: 8, loss: 0.0015\n",
312+
"episode: 929, time: 9, loss: 0.0021\n",
313+
"episode: 939, time: 9, loss: 0.0015\n",
314+
"episode: 949, time: 9, loss: 0.0016\n",
315+
"episode: 959, time: 9, loss: 0.0013\n",
316+
"episode: 969, time: 12, loss: 0.0029\n",
317+
"episode: 979, time: 7, loss: 0.0016\n",
318+
"episode: 989, time: 7, loss: 0.0012\n",
319+
"episode: 999, time: 9, loss: 0.0013\n"
320+
]
321+
}
322+
],
323+
"source": [
324+
"deep_q_learning(1000)"
325+
]
326+
},
327+
{
328+
"cell_type": "code",
329+
"execution_count": null,
330+
"metadata": {
331+
"collapsed": true
332+
},
333+
"outputs": [],
334+
"source": []
335+
}
336+
],
337+
"metadata": {
338+
"anaconda-cloud": {},
339+
"kernelspec": {
340+
"display_name": "Python 2",
341+
"language": "python",
342+
"name": "python2"
343+
},
344+
"language_info": {
345+
"codemirror_mode": {
346+
"name": "ipython",
347+
"version": 2
348+
},
349+
"file_extension": ".py",
350+
"mimetype": "text/x-python",
351+
"name": "python",
352+
"nbconvert_exporter": "python",
353+
"pygments_lexer": "ipython2",
354+
"version": "2.7.13"
355+
}
356+
},
357+
"nbformat": 4,
358+
"nbformat_minor": 1
359+
}

0 commit comments

Comments
 (0)