Reinforcement Learning : Apprendre par l'Erreur

Q(s, a) = r + γ · max_{a'} Q(s', a')

import numpy as np

Q = np.zeros((16, 4))  # 16 états, 4 actions
alpha, gamma, epsilon = 0.1, 0.99, 0.1

def choisir_action(state):
    if np.random.random() < epsilon:
        return np.random.randint(4)  # Explorer
    return np.argmax(Q[state])       # Exploiter

def mise_a_jour_q(state, action, reward, next_state):
    q_actuel = Q[state, action]
    q_cible  = reward + gamma * np.max(Q[next_state])
    Q[state, action] = q_actuel + alpha * (q_cible - q_actuel)

import torch, torch.nn as nn, random
from collections import deque

class QNetwork(nn.Module):
    def __init__(self, obs_size, n_actions):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(obs_size, 128), nn.ReLU(),
            nn.Linear(128, 64), nn.ReLU(),
            nn.Linear(64, n_actions)
        )
    def forward(self, x): return self.net(x)

class ReplayBuffer:
    def __init__(self, capacity=10000):
        self.buffer = deque(maxlen=capacity)
    def push(self, *args): self.buffer.append(args)
    def sample(self, n): return random.sample(self.buffer, n)
    def __len__(self): return len(self.buffer)

import gymnasium as gym
import torch, torch.nn as nn, torch.optim as optim, random

env = gym.make("CartPole-v1")
obs_size = env.observation_space.shape[0]  # 4
n_actions = env.action_space.n             # 2

q_net = QNetwork(obs_size, n_actions)
target_net = QNetwork(obs_size, n_actions)
target_net.load_state_dict(q_net.state_dict())
print(f"Réseau : {obs_size} entrées → {n_actions} actions Q-values")

optimizer = optim.Adam(q_net.parameters(), lr=1e-3)
buffer = ReplayBuffer(10000)
epsilon, gamma = 1.0, 0.99

for episode in range(300):
    state, _ = env.reset()
    total_reward = 0

    while True:
        # Epsilon-greedy
        if random.random() < epsilon:
            action = env.action_space.sample()
        else:
            with torch.no_grad():
                action = q_net(torch.FloatTensor(state)).argmax().item()

        next_state, reward, terminated, truncated, _ = env.step(action)
        done = terminated or truncated
        buffer.push(state, action, reward, next_state, done)
        state = next_state
        total_reward += reward

        # Entraînement
        if len(buffer) >= 64:
            batch = buffer.sample(64)
            s, a, r, ns, d = [torch.FloatTensor(x) for x in zip(*batch)]
            a = a.long()
            curr_q = q_net(s).gather(1, a.unsqueeze(1)).squeeze()
            with torch.no_grad():
                next_q = target_net(ns).max(1)[0]
                tgt_q  = r + gamma * next_q * (1 - d)
            loss = nn.MSELoss()(curr_q, tgt_q)
            optimizer.zero_grad(); loss.backward(); optimizer.step()

        if done: break

    epsilon = max(0.01, epsilon * 0.995)
    if episode % 50 == 0:
        print(f"Episode {episode:3d} | Reward: {total_reward:.0f} | ε: {epsilon:.3f}")