{"cells": [{"cell_type": "markdown", "id": "017408fb", "metadata": {"papermill": {"duration": 0.003265, "end_time": "2023-10-12T02:32:22.601006", "exception": false, "start_time": "2023-10-12T02:32:22.597741", "status": "completed"}, "tags": []}, "source": ["\n", "# How to train a Deep Q Network\n", "\n", "* **Author:** Lightning.ai\n", "* **License:** CC BY-SA\n", "* **Generated:** 2023-10-12T02:31:55.130696\n", "\n", "Main takeaways:\n", "\n", "1. RL has the same flow as previous models we have seen, with a few additions\n", "2. Handle unsupervised learning by using an IterableDataset where the dataset itself is constantly updated during training\n", "3. Each training step carries has the agent taking an action in the environment and storing the experience in the IterableDataset\n", "\n", "\n", "---\n", "Open in [{height=\"20px\" width=\"117px\"}](https://colab.research.google.com/github/PytorchLightning/lightning-tutorials/blob/publication/.notebooks/lightning_examples/reinforce-learning-DQN.ipynb)\n", "\n", "Give us a \u2b50 [on Github](https://www.github.com/Lightning-AI/lightning/)\n", "| Check out [the documentation](https://pytorch-lightning.readthedocs.io/en/stable/)\n", "| Join us [on Slack](https://www.pytorchlightning.ai/community)"]}, {"cell_type": "markdown", "id": "022f9a20", "metadata": {"papermill": {"duration": 0.002439, "end_time": "2023-10-12T02:32:22.606166", "exception": false, "start_time": "2023-10-12T02:32:22.603727", "status": "completed"}, "tags": []}, "source": ["## Setup\n", "This notebook requires some packages besides pytorch-lightning."]}, {"cell_type": "code", "execution_count": 1, "id": "af6ab99e", "metadata": {"colab": {}, "colab_type": "code", "execution": {"iopub.execute_input": "2023-10-12T02:32:22.612363Z", "iopub.status.busy": "2023-10-12T02:32:22.612193Z", "iopub.status.idle": "2023-10-12T02:32:26.226744Z", "shell.execute_reply": "2023-10-12T02:32:26.225392Z"}, "id": "LfrJLKPFyhsK", "lines_to_next_cell": 0, "papermill": {"duration": 3.620149, "end_time": "2023-10-12T02:32:26.228791", "exception": false, "start_time": "2023-10-12T02:32:22.608642", "status": "completed"}, "tags": []}, "outputs": [{"name": "stdout", "output_type": "stream", "text": ["\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n", "\u001b[0m"]}], "source": ["! pip install --quiet \"torchmetrics>=0.7, <1.3\" \"torch>=1.8.1, <2.1.0\" \"seaborn\" \"ipython[notebook]>=8.0.0, <8.17.0\" \"urllib3\" \"setuptools>=68.0.0, <68.3.0\" \"pygame\" \"pandas\" \"matplotlib>=3.0.0, <3.9.0\" \"pytorch-lightning>=1.4, <2.1.0\" \"lightning>=2.0.0\" \"gym <0.24\""]}, {"cell_type": "code", "execution_count": 2, "id": "39abd144", "metadata": {"execution": {"iopub.execute_input": "2023-10-12T02:32:26.236161Z", "iopub.status.busy": "2023-10-12T02:32:26.235792Z", "iopub.status.idle": "2023-10-12T02:32:29.015454Z", "shell.execute_reply": "2023-10-12T02:32:29.014202Z"}, "papermill": {"duration": 2.785715, "end_time": "2023-10-12T02:32:29.017517", "exception": false, "start_time": "2023-10-12T02:32:26.231802", "status": "completed"}, "tags": []}, "outputs": [{"name": "stderr", "output_type": "stream", "text": ["/tmp/ipykernel_598/94481177.py:10: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display\n", " from IPython.core.display import display\n", "/usr/local/lib/python3.10/dist-packages/lightning_utilities/core/imports.py:13: DeprecationWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html\n", " import pkg_resources\n", "/usr/local/lib/python3.10/dist-packages/lightning_fabric/__init__.py:36: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('lightning_fabric')`.\n", "Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages\n", " __import__(\"pkg_resources\").declare_namespace(__name__)\n", "/usr/local/lib/python3.10/dist-packages/torchmetrics/utilities/imports.py:24: DeprecationWarning: distutils Version classes are deprecated. Use packaging.version instead.\n", " _PYTHON_LOWER_3_8 = LooseVersion(_PYTHON_VERSION) < LooseVersion(\"3.8\")\n"]}, {"name": "stderr", "output_type": "stream", "text": ["/usr/local/lib/python3.10/dist-packages/pytorch_lightning/__init__.py:36: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('pytorch_lightning')`.\n", "Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages\n", " __import__(\"pkg_resources\").declare_namespace(__name__)\n"]}], "source": ["import os\n", "from collections import OrderedDict, deque, namedtuple\n", "from typing import Iterator, List, Tuple\n", "\n", "import gym\n", "import numpy as np\n", "import pandas as pd\n", "import seaborn as sn\n", "import torch\n", "from IPython.core.display import display\n", "from pytorch_lightning import LightningModule, Trainer\n", "from pytorch_lightning.loggers import CSVLogger\n", "from torch import Tensor, nn\n", "from torch.optim import Adam, Optimizer\n", "from torch.utils.data import DataLoader\n", "from torch.utils.data.dataset import IterableDataset\n", "\n", "PATH_DATASETS = os.environ.get(\"PATH_DATASETS\", \".\")"]}, {"cell_type": "code", "execution_count": 3, "id": "43d268bf", "metadata": {"execution": {"iopub.execute_input": "2023-10-12T02:32:29.024808Z", "iopub.status.busy": "2023-10-12T02:32:29.024435Z", "iopub.status.idle": "2023-10-12T02:32:29.030483Z", "shell.execute_reply": "2023-10-12T02:32:29.029496Z"}, "papermill": {"duration": 0.012004, "end_time": "2023-10-12T02:32:29.032470", "exception": false, "start_time": "2023-10-12T02:32:29.020466", "status": "completed"}, "tags": []}, "outputs": [], "source": ["class DQN(nn.Module):\n", " def __init__(self, obs_size: int, n_actions: int, hidden_size: int = 128):\n", " \"\"\"Simple MLP network.\n", "\n", " Args:\n", " obs_size: observation/state size of the environment\n", " n_actions: number of discrete actions available in the environment\n", " hidden_size: size of hidden layers\n", " \"\"\"\n", " super().__init__()\n", " self.net = nn.Sequential(\n", " nn.Linear(obs_size, hidden_size),\n", " nn.ReLU(),\n", " nn.Linear(hidden_size, n_actions),\n", " )\n", "\n", " def forward(self, x):\n", " return self.net(x.float())"]}, {"cell_type": "markdown", "id": "b54c1545", "metadata": {"papermill": {"duration": 0.003517, "end_time": "2023-10-12T02:32:29.038793", "exception": false, "start_time": "2023-10-12T02:32:29.035276", "status": "completed"}, "tags": []}, "source": ["### Memory"]}, {"cell_type": "code", "execution_count": 4, "id": "52218c62", "metadata": {"execution": {"iopub.execute_input": "2023-10-12T02:32:29.045488Z", "iopub.status.busy": "2023-10-12T02:32:29.045304Z", "iopub.status.idle": "2023-10-12T02:32:29.049602Z", "shell.execute_reply": "2023-10-12T02:32:29.048617Z"}, "papermill": {"duration": 0.009457, "end_time": "2023-10-12T02:32:29.051006", "exception": false, "start_time": "2023-10-12T02:32:29.041549", "status": "completed"}, "tags": []}, "outputs": [], "source": ["\n", "# Named tuple for storing experience steps gathered in training\n", "Experience = namedtuple(\n", " \"Experience\",\n", " field_names=[\"state\", \"action\", \"reward\", \"done\", \"new_state\"],\n", ")"]}, {"cell_type": "code", "execution_count": 5, "id": "ea2c7d65", "metadata": {"execution": {"iopub.execute_input": "2023-10-12T02:32:29.057722Z", "iopub.status.busy": "2023-10-12T02:32:29.057486Z", "iopub.status.idle": "2023-10-12T02:32:29.065309Z", "shell.execute_reply": "2023-10-12T02:32:29.064253Z"}, "papermill": {"duration": 0.012862, "end_time": "2023-10-12T02:32:29.066667", "exception": false, "start_time": "2023-10-12T02:32:29.053805", "status": "completed"}, "tags": []}, "outputs": [], "source": ["class ReplayBuffer:\n", " \"\"\"Replay Buffer for storing past experiences allowing the agent to learn from them.\n", "\n", " Args:\n", " capacity: size of the buffer\n", " \"\"\"\n", "\n", " def __init__(self, capacity: int) -> None:\n", " self.buffer = deque(maxlen=capacity)\n", "\n", " def __len__(self) -> None:\n", " return len(self.buffer)\n", "\n", " def append(self, experience: Experience) -> None:\n", " \"\"\"Add experience to the buffer.\n", "\n", " Args:\n", " experience: tuple (state, action, reward, done, new_state)\n", " \"\"\"\n", " self.buffer.append(experience)\n", "\n", " def sample(self, batch_size: int) -> Tuple:\n", " indices = np.random.choice(len(self.buffer), batch_size, replace=False)\n", " states, actions, rewards, dones, next_states = zip(*(self.buffer[idx] for idx in indices))\n", "\n", " return (\n", " np.array(states),\n", " np.array(actions),\n", " np.array(rewards, dtype=np.float32),\n", " np.array(dones, dtype=bool),\n", " np.array(next_states),\n", " )"]}, {"cell_type": "code", "execution_count": 6, "id": "1bf45d19", "metadata": {"execution": {"iopub.execute_input": "2023-10-12T02:32:29.073593Z", "iopub.status.busy": "2023-10-12T02:32:29.073354Z", "iopub.status.idle": "2023-10-12T02:32:29.079728Z", "shell.execute_reply": "2023-10-12T02:32:29.078683Z"}, "lines_to_next_cell": 2, "papermill": {"duration": 0.011456, "end_time": "2023-10-12T02:32:29.081004", "exception": false, "start_time": "2023-10-12T02:32:29.069548", "status": "completed"}, "tags": []}, "outputs": [], "source": ["class RLDataset(IterableDataset):\n", " \"\"\"Iterable Dataset containing the ExperienceBuffer which will be updated with new experiences during training.\n", "\n", " Args:\n", " buffer: replay buffer\n", " sample_size: number of experiences to sample at a time\n", " \"\"\"\n", "\n", " def __init__(self, buffer: ReplayBuffer, sample_size: int = 200) -> None:\n", " self.buffer = buffer\n", " self.sample_size = sample_size\n", "\n", " def __iter__(self) -> Iterator[Tuple]:\n", " states, actions, rewards, dones, new_states = self.buffer.sample(self.sample_size)\n", " for i in range(len(dones)):\n", " yield states[i], actions[i], rewards[i], dones[i], new_states[i]"]}, {"cell_type": "markdown", "id": "4590f93f", "metadata": {"lines_to_next_cell": 2, "papermill": {"duration": 0.002819, "end_time": "2023-10-12T02:32:29.086659", "exception": false, "start_time": "2023-10-12T02:32:29.083840", "status": "completed"}, "tags": []}, "source": ["### Agent"]}, {"cell_type": "code", "execution_count": 7, "id": "5fed98cf", "metadata": {"execution": {"iopub.execute_input": "2023-10-12T02:32:29.093635Z", "iopub.status.busy": "2023-10-12T02:32:29.093396Z", "iopub.status.idle": "2023-10-12T02:32:29.104106Z", "shell.execute_reply": "2023-10-12T02:32:29.103165Z"}, "lines_to_next_cell": 2, "papermill": {"duration": 0.016019, "end_time": "2023-10-12T02:32:29.105495", "exception": false, "start_time": "2023-10-12T02:32:29.089476", "status": "completed"}, "tags": []}, "outputs": [], "source": ["class Agent:\n", " def __init__(self, env: gym.Env, replay_buffer: ReplayBuffer) -> None:\n", " \"\"\"Base Agent class handling the interaction with the environment.\n", "\n", " Args:\n", " env: training environment\n", " replay_buffer: replay buffer storing experiences\n", " \"\"\"\n", " self.env = env\n", " self.replay_buffer = replay_buffer\n", " self.reset()\n", " self.state = self.env.reset()\n", "\n", " def reset(self) -> None:\n", " \"\"\"Resents the environment and updates the state.\"\"\"\n", " self.state = self.env.reset()\n", "\n", " def get_action(self, net: nn.Module, epsilon: float, device: str) -> int:\n", " \"\"\"Using the given network, decide what action to carry out using an epsilon-greedy policy.\n", "\n", " Args:\n", " net: DQN network\n", " epsilon: value to determine likelihood of taking a random action\n", " device: current device\n", "\n", " Returns:\n", " action\n", " \"\"\"\n", " if np.random.random() < epsilon:\n", " action = self.env.action_space.sample()\n", " else:\n", " state = torch.tensor([self.state])\n", "\n", " if device not in [\"cpu\"]:\n", " state = state.cuda(device)\n", "\n", " q_values = net(state)\n", " _, action = torch.max(q_values, dim=1)\n", " action = int(action.item())\n", "\n", " return action\n", "\n", " @torch.no_grad()\n", " def play_step(\n", " self,\n", " net: nn.Module,\n", " epsilon: float = 0.0,\n", " device: str = \"cpu\",\n", " ) -> Tuple[float, bool]:\n", " \"\"\"Carries out a single interaction step between the agent and the environment.\n", "\n", " Args:\n", " net: DQN network\n", " epsilon: value to determine likelihood of taking a random action\n", " device: current device\n", "\n", " Returns:\n", " reward, done\n", " \"\"\"\n", " action = self.get_action(net, epsilon, device)\n", "\n", " # do step in the environment\n", " # So, in the deprecated version of gym, the env.step() has 4 values unpacked which is\n", " # obs, reward, done, info = env.step(action)\n", " # In the latest version of gym, the step() function returns back an additional variable which is truncated.\n", " # obs, reward, terminated, truncated, info = env.step(action)\n", " new_state, reward, done, _ = self.env.step(action)\n", "\n", " exp = Experience(self.state, action, reward, done, new_state)\n", "\n", " self.replay_buffer.append(exp)\n", "\n", " self.state = new_state\n", " if done:\n", " self.reset()\n", " return reward, done"]}, {"cell_type": "markdown", "id": "efabc3b9", "metadata": {"lines_to_next_cell": 2, "papermill": {"duration": 0.002849, "end_time": "2023-10-12T02:32:29.111214", "exception": false, "start_time": "2023-10-12T02:32:29.108365", "status": "completed"}, "tags": []}, "source": ["### DQN Lightning Module"]}, {"cell_type": "code", "execution_count": 8, "id": "01661227", "metadata": {"execution": {"iopub.execute_input": "2023-10-12T02:32:29.118119Z", "iopub.status.busy": "2023-10-12T02:32:29.117874Z", "iopub.status.idle": "2023-10-12T02:32:29.146533Z", "shell.execute_reply": "2023-10-12T02:32:29.145595Z"}, "papermill": {"duration": 0.033675, "end_time": "2023-10-12T02:32:29.147749", "exception": false, "start_time": "2023-10-12T02:32:29.114074", "status": "completed"}, "tags": []}, "outputs": [], "source": ["class DQNLightning(LightningModule):\n", " def __init__(\n", " self,\n", " batch_size: int = 16,\n", " lr: float = 1e-2,\n", " env: str = \"CartPole-v0\",\n", " gamma: float = 0.99,\n", " sync_rate: int = 10,\n", " replay_size: int = 1000,\n", " warm_start_size: int = 1000,\n", " eps_last_frame: int = 1000,\n", " eps_start: float = 1.0,\n", " eps_end: float = 0.01,\n", " episode_length: int = 200,\n", " warm_start_steps: int = 1000,\n", " ) -> None:\n", " \"\"\"Basic DQN Model.\n", "\n", " Args:\n", " batch_size: size of the batches\")\n", " lr: learning rate\n", " env: gym environment tag\n", " gamma: discount factor\n", " sync_rate: how many frames do we update the target network\n", " replay_size: capacity of the replay buffer\n", " warm_start_size: how many samples do we use to fill our buffer at the start of training\n", " eps_last_frame: what frame should epsilon stop decaying\n", " eps_start: starting value of epsilon\n", " eps_end: final value of epsilon\n", " episode_length: max length of an episode\n", " warm_start_steps: max episode reward in the environment\n", " \"\"\"\n", " super().__init__()\n", " self.save_hyperparameters()\n", "\n", " self.env = gym.make(self.hparams.env)\n", " obs_size = self.env.observation_space.shape[0]\n", " n_actions = self.env.action_space.n\n", "\n", " self.net = DQN(obs_size, n_actions)\n", " self.target_net = DQN(obs_size, n_actions)\n", "\n", " self.buffer = ReplayBuffer(self.hparams.replay_size)\n", " self.agent = Agent(self.env, self.buffer)\n", " self.total_reward = 0\n", " self.episode_reward = 0\n", " self.populate(self.hparams.warm_start_steps)\n", "\n", " def populate(self, steps: int = 1000) -> None:\n", " \"\"\"Carries out several random steps through the environment to initially fill up the replay buffer with\n", " experiences.\n", "\n", " Args:\n", " steps: number of random steps to populate the buffer with\n", " \"\"\"\n", " for _ in range(steps):\n", " self.agent.play_step(self.net, epsilon=1.0)\n", "\n", " def forward(self, x: Tensor) -> Tensor:\n", " \"\"\"Passes in a state x through the network and gets the q_values of each action as an output.\n", "\n", " Args:\n", " x: environment state\n", "\n", " Returns:\n", " q values\n", " \"\"\"\n", " output = self.net(x)\n", " return output\n", "\n", " def dqn_mse_loss(self, batch: Tuple[Tensor, Tensor]) -> Tensor:\n", " \"\"\"Calculates the mse loss using a mini batch from the replay buffer.\n", "\n", " Args:\n", " batch: current mini batch of replay data\n", "\n", " Returns:\n", " loss\n", " \"\"\"\n", " states, actions, rewards, dones, next_states = batch\n", "\n", " state_action_values = self.net(states).gather(1, actions.long().unsqueeze(-1)).squeeze(-1)\n", "\n", " with torch.no_grad():\n", " next_state_values = self.target_net(next_states).max(1)[0]\n", " next_state_values[dones] = 0.0\n", " next_state_values = next_state_values.detach()\n", "\n", " expected_state_action_values = next_state_values * self.hparams.gamma + rewards\n", "\n", " return nn.MSELoss()(state_action_values, expected_state_action_values)\n", "\n", " def get_epsilon(self, start: int, end: int, frames: int) -> float:\n", " if self.global_step > frames:\n", " return end\n", " return start - (self.global_step / frames) * (start - end)\n", "\n", " def training_step(self, batch: Tuple[Tensor, Tensor], nb_batch) -> OrderedDict:\n", " \"\"\"Carries out a single step through the environment to update the replay buffer. Then calculates loss based on\n", " the minibatch received.\n", "\n", " Args:\n", " batch: current mini batch of replay data\n", " nb_batch: batch number\n", "\n", " Returns:\n", " Training loss and log metrics\n", " \"\"\"\n", " device = self.get_device(batch)\n", " epsilon = self.get_epsilon(self.hparams.eps_start, self.hparams.eps_end, self.hparams.eps_last_frame)\n", " self.log(\"epsilon\", epsilon)\n", "\n", " # step through environment with agent\n", " reward, done = self.agent.play_step(self.net, epsilon, device)\n", " self.episode_reward += reward\n", " self.log(\"episode reward\", self.episode_reward)\n", "\n", " # calculates training loss\n", " loss = self.dqn_mse_loss(batch)\n", "\n", " if done:\n", " self.total_reward = self.episode_reward\n", " self.episode_reward = 0\n", "\n", " # Soft update of target network\n", " if self.global_step % self.hparams.sync_rate == 0:\n", " self.target_net.load_state_dict(self.net.state_dict())\n", "\n", " self.log_dict(\n", " {\n", " \"reward\": reward,\n", " \"train_loss\": loss,\n", " }\n", " )\n", " self.log(\"total_reward\", self.total_reward, prog_bar=True)\n", " self.log(\"steps\", self.global_step, logger=False, prog_bar=True)\n", "\n", " return loss\n", "\n", " def configure_optimizers(self) -> List[Optimizer]:\n", " \"\"\"Initialize Adam optimizer.\"\"\"\n", " optimizer = Adam(self.net.parameters(), lr=self.hparams.lr)\n", " return optimizer\n", "\n", " def __dataloader(self) -> DataLoader:\n", " \"\"\"Initialize the Replay Buffer dataset used for retrieving experiences.\"\"\"\n", " dataset = RLDataset(self.buffer, self.hparams.episode_length)\n", " dataloader = DataLoader(\n", " dataset=dataset,\n", " batch_size=self.hparams.batch_size,\n", " )\n", " return dataloader\n", "\n", " def train_dataloader(self) -> DataLoader:\n", " \"\"\"Get train loader.\"\"\"\n", " return self.__dataloader()\n", "\n", " def get_device(self, batch) -> str:\n", " \"\"\"Retrieve device currently being used by minibatch.\"\"\"\n", " return batch[0].device.index if self.on_gpu else \"cpu\""]}, {"cell_type": "markdown", "id": "ddff04fb", "metadata": {"papermill": {"duration": 0.002878, "end_time": "2023-10-12T02:32:29.153668", "exception": false, "start_time": "2023-10-12T02:32:29.150790", "status": "completed"}, "tags": []}, "source": ["### Trainer"]}, {"cell_type": "code", "execution_count": 9, "id": "a7a082fe", "metadata": {"execution": {"iopub.execute_input": "2023-10-12T02:32:29.160921Z", "iopub.status.busy": "2023-10-12T02:32:29.160560Z", "iopub.status.idle": "2023-10-12T02:32:39.231606Z", "shell.execute_reply": "2023-10-12T02:32:39.230728Z"}, "papermill": {"duration": 10.077523, "end_time": "2023-10-12T02:32:39.234065", "exception": false, "start_time": "2023-10-12T02:32:29.156542", "status": "completed"}, "tags": []}, "outputs": [{"name": "stderr", "output_type": "stream", "text": ["/usr/local/lib/python3.10/dist-packages/gym/envs/registration.py:505: UserWarning: \u001b[33mWARN: The environment CartPole-v0 is out of date. You should consider upgrading to version `v1` with the environment ID `CartPole-v1`.\u001b[0m\n", " logger.warn(\n"]}, {"name": "stderr", "output_type": "stream", "text": ["GPU available: True (cuda), used: True\n"]}, {"name": "stderr", "output_type": "stream", "text": ["TPU available: False, using: 0 TPU cores\n"]}, {"name": "stderr", "output_type": "stream", "text": ["IPU available: False, using: 0 IPUs\n"]}, {"name": "stderr", "output_type": "stream", "text": ["HPU available: False, using: 0 HPUs\n"]}, {"name": "stderr", "output_type": "stream", "text": ["You are using a CUDA device ('NVIDIA GeForce RTX 3090') that has Tensor Cores. To properly utilize them, you should set `torch.set_float32_matmul_precision('medium' | 'high')` which will trade-off precision for performance. For more details, read https://pytorch.org/docs/stable/generated/torch.set_float32_matmul_precision.html#torch.set_float32_matmul_precision\n"]}, {"name": "stderr", "output_type": "stream", "text": ["LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1]\n"]}, {"name": "stderr", "output_type": "stream", "text": ["\n", " | Name | Type | Params\n", "------------------------------------\n", "0 | net | DQN | 898 \n", "1 | target_net | DQN | 898 \n", "------------------------------------\n", "1.8 K Trainable params\n", "0 Non-trainable params\n", "1.8 K Total params\n", "0.007 Total estimated model params size (MB)\n"]}, {"name": "stderr", "output_type": "stream", "text": ["/usr/local/lib/python3.10/dist-packages/pytorch_lightning/trainer/connectors/data_connector.py:442: PossibleUserWarning: The dataloader, train_dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 64 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n", " rank_zero_warn(\n"]}, {"data": {"application/vnd.jupyter.widget-view+json": {"model_id": "92023abcd7774a91896fc743d933e5c5", "version_major": 2, "version_minor": 0}, "text/plain": ["Training: 0it [00:00, ?it/s]"]}, "metadata": {}, "output_type": "display_data"}, {"name": "stderr", "output_type": "stream", "text": ["/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/collate.py:175: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index\n", " return torch.as_tensor(batch)\n"]}, {"name": "stderr", "output_type": "stream", "text": ["/usr/local/lib/python3.10/dist-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:212: UserWarning: You called `self.log('total_reward', ...)` in your `training_step` but the value needs to be floating point. Converting it to torch.float32.\n", " warning_cache.warn(\n", "/usr/local/lib/python3.10/dist-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:212: UserWarning: You called `self.log('steps', ...)` in your `training_step` but the value needs to be floating point. Converting it to torch.float32.\n", " warning_cache.warn(\n"]}, {"name": "stderr", "output_type": "stream", "text": ["/tmp/ipykernel_598/1958637465.py:32: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ../torch/csrc/utils/tensor_new.cpp:245.)\n", " state = torch.tensor([self.state])\n"]}, {"name": "stderr", "output_type": "stream", "text": ["`Trainer.fit` stopped: `max_epochs=150` reached.\n"]}], "source": ["\n", "model = DQNLightning()\n", "\n", "trainer = Trainer(\n", " accelerator=\"auto\",\n", " devices=1 if torch.cuda.is_available() else None, # limiting got iPython runs\n", " max_epochs=150,\n", " val_check_interval=50,\n", " logger=CSVLogger(save_dir=\"logs/\"),\n", ")\n", "\n", "trainer.fit(model)"]}, {"cell_type": "code", "execution_count": 10, "id": "2b2a4ec6", "metadata": {"execution": {"iopub.execute_input": "2023-10-12T02:32:39.253630Z", "iopub.status.busy": "2023-10-12T02:32:39.253435Z", "iopub.status.idle": "2023-10-12T02:32:39.593819Z", "shell.execute_reply": "2023-10-12T02:32:39.592879Z"}, "papermill": {"duration": 0.35152, "end_time": "2023-10-12T02:32:39.595230", "exception": false, "start_time": "2023-10-12T02:32:39.243710", "status": "completed"}, "tags": []}, "outputs": [{"data": {"text/html": ["\n", "\n", "
\n", " \n", " \n", " | \n", " epsilon | \n", " episode reward | \n", " reward | \n", " train_loss | \n", " total_reward | \n", "
\n", " \n", " | epoch | \n", " | \n", " | \n", " | \n", " | \n", " | \n", "
\n", " \n", " \n", " \n", " | 3 | \n", " 0.95149 | \n", " 22.0 | \n", " 1.0 | \n", " 0.263845 | \n", " 9.0 | \n", "
\n", " \n", " | 7 | \n", " 0.90199 | \n", " 18.0 | \n", " 1.0 | \n", " 1.852319 | \n", " 18.0 | \n", "
\n", " \n", " | 11 | \n", " 0.85249 | \n", " 11.0 | \n", " 1.0 | \n", " 0.130760 | \n", " 25.0 | \n", "
\n", " \n", " | 15 | \n", " 0.80299 | \n", " 17.0 | \n", " 1.0 | \n", " 25.509975 | \n", " 15.0 | \n", "
\n", " \n", " | 19 | \n", " 0.75349 | \n", " 22.0 | \n", " 1.0 | \n", " 55.599968 | \n", " 19.0 | \n", "
\n", " \n", "
\n", "