PyTorch 深度学习实战（23）：多任务强化学习（Multi-Task RL）之扩展

之前的PyTorch 深度学习实战（23）：多任务强化学习（Multi-Task RL)总结扩展运用代码如下：

import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np
from torch.distributions import Normal
from torch.amp import autocast, GradScaler
from metaworld.envs import ALL_V2_ENVIRONMENTS_GOAL_OBSERVABLE
import time
from collections import deque# ================== 配置参数 ==================
class MultiTaskPPOConfig:# 任务配置task_names = ['reach-v2-goal-observable','push-v2-goal-observable','pick-place-v2-goal-observable']num_tasks = 3# 网络架构shared_dim = 512task_specific_dim = 256meta_controller_dim = 128shared_layers = 2task_specific_layers = 1# 训练参数lr = 5e-5meta_lr = 1e-5gamma = 0.99gae_lambda = 0.97clip_epsilon = 0.15ppo_epochs = 5batch_size = 4096max_episodes = 10000max_steps = 200grad_clip = 0.5entropy_coef = 0.1# 探索参数initial_std = 1.5min_std = 0.2std_decay = 0.999# 课程学习安排curriculum_schedule = {0: ['reach-v2-goal-observable'],1000: ['reach-v2-goal-observable', 'push-v2-goal-observable'],3000: ['push-v2-goal-observable', 'pick-place-v2-goal-observable'],6000: ['reach-v2-goal-observable', 'push-v2-goal-observable', 'pick-place-v2-goal-observable']}# 监控配置log_interval = 50eval_interval = 500eval_episodes = 10device = torch.device("cuda" if torch.cuda.is_available() else "cpu")# ================== MetaController ==================
class MetaController(nn.Module):def __init__(self, num_tasks, state_dim):super().__init__()self.net = nn.Sequential(nn.Linear(state_dim, MultiTaskPPOConfig.meta_controller_dim),nn.LayerNorm(MultiTaskPPOConfig.meta_controller_dim),nn.GELU(),nn.Linear(MultiTaskPPOConfig.meta_controller_dim, num_tasks))# 初始化参数for layer in self.net:if isinstance(layer, nn.Linear):nn.init.orthogonal_(layer.weight, gain=0.01)nn.init.constant_(layer.bias, 0.0)def forward(self, state):logits = self.net(state)return torch.softmax(logits, -1), logits# ================== 共享策略网络 ==================
class SharedPolicy(nn.Module):def __init__(self, state_dim, action_dim):super().__init__()self.action_dim = action_dimself.current_std = MultiTaskPPOConfig.initial_std# 共享网络层self.shared_net = nn.Sequential(nn.Linear(state_dim, MultiTaskPPOConfig.shared_dim),nn.LayerNorm(MultiTaskPPOConfig.shared_dim),nn.GELU(),nn.Linear(MultiTaskPPOConfig.shared_dim, MultiTaskPPOConfig.shared_dim),nn.GELU())# 多任务头部self.task_heads = nn.ModuleList()self.value_heads = nn.ModuleList()for _ in range(MultiTaskPPOConfig.num_tasks):# 动作头task_head = nn.Sequential(nn.Linear(MultiTaskPPOConfig.shared_dim, MultiTaskPPOConfig.task_specific_dim),nn.GELU(),nn.Linear(MultiTaskPPOConfig.task_specific_dim, action_dim))self.task_heads.append(task_head)# 值函数头value_head = nn.Sequential(nn.Linear(MultiTaskPPOConfig.shared_dim, MultiTaskPPOConfig.task_specific_dim),nn.GELU(),nn.Linear(MultiTaskPPOConfig.task_specific_dim, 1))self.value_heads.append(value_head)# 可学习的对数标准差self.log_std = nn.Parameter(torch.zeros(1, action_dim))# 初始化参数self._init_weights()def _init_weights(self):for head in self.task_heads:for layer in head:if isinstance(layer, nn.Linear):nn.init.orthogonal_(layer.weight, gain=0.01)nn.init.constant_(layer.bias, 0.0)for head in self.value_heads:for layer in head:if isinstance(layer, nn.Linear):nn.init.orthogonal_(layer.weight, gain=1.0)nn.init.constant_(layer.bias, 0.0)def decay_action_std(self):"""衰减动作标准差"""self.current_std = max(self.current_std * MultiTaskPPOConfig.std_decay,MultiTaskPPOConfig.min_std)def forward(self, states, task_ids):# 确保输入是float32states = states.float() if states.dtype != torch.float32 else statesshared_features = self.shared_net(states)batch_size = states.size(0)# 初始化输出张量action_means = torch.zeros(batch_size, self.action_dim,dtype=torch.float32,device=states.device)action_stds = torch.exp(self.log_std).expand(batch_size, -1) * self.current_stdvalues = torch.zeros(batch_size, 1,dtype=torch.float32,device=states.device)unique_task_ids = torch.unique(task_ids)for task_id_tensor in unique_task_ids:task_id = task_id_tensor.item()mask = (task_ids == task_id_tensor)if not mask.any():continueselected_features = shared_features[mask]# 计算任务特定输出with autocast(device_type=states.device.type, enabled=False):  # 禁用混合精度task_action = self.task_heads[task_id](selected_features.float())task_value = self.value_heads[task_id](selected_features.float())action_means[mask] = task_actionvalues[mask] = task_valuereturn action_means, action_stds, values# ================== 训练系统 ==================
class EnhancedMultiTaskPPOTrainer:def __init__(self):# 初始化多任务环境self.envs = []self.state_dim = Noneself.action_dim = None# 验证环境并获取维度for task_name in MultiTaskPPOConfig.task_names:env = ALL_V2_ENVIRONMENTS_GOAL_OBSERVABLE[task_name]()obs, _ = env.reset()if self.state_dim is None:self.state_dim = obs.shape[0]self.action_dim = env.action_space.shape[0]else:assert obs.shape[0] == self.state_dim, f"状态维度不一致: {task_name}"self.envs.append(env)# 初始化策略网络self.policy = SharedPolicy(self.state_dim, self.action_dim).to(MultiTaskPPOConfig.device)self.optimizer = optim.AdamW(self.policy.parameters(), lr=MultiTaskPPOConfig.lr)self.scheduler = optim.lr_scheduler.CosineAnnealingLR(self.optimizer,T_max=MultiTaskPPOConfig.max_episodes,eta_min=1e-6)self.scaler = GradScaler(enabled=MultiTaskPPOConfig.device.type == 'cuda')# 初始化MetaControllerself.meta_controller = MetaController(MultiTaskPPOConfig.num_tasks,self.state_dim).to(MultiTaskPPOConfig.device)self.meta_optimizer = optim.Adam(self.meta_controller.parameters(),lr=MultiTaskPPOConfig.meta_lr)# 初始化经验回放缓冲self.buffer = deque(maxlen=MultiTaskPPOConfig.max_steps)# 课程学习状态self.current_phase = 0self.phase_thresholds = sorted(MultiTaskPPOConfig.curriculum_schedule.keys())# 训练统计self.episode_rewards = {i: deque(maxlen=100) for i in range(MultiTaskPPOConfig.num_tasks)}self.episode_lengths = {i: deque(maxlen=100) for i in range(MultiTaskPPOConfig.num_tasks)}self.meta_data = {'states': [],'chosen_tasks': [],'rewards': []}# 评估统计self.eval_rewards = {i: [] for i in range(MultiTaskPPOConfig.num_tasks)}self.eval_success = {i: [] for i in range(MultiTaskPPOConfig.num_tasks)}def get_current_tasks(self, episode):"""获取当前课程阶段的任务列表"""if len(self.phase_thresholds) > 1 and self.current_phase < len(self.phase_thresholds) - 1:if episode >= self.phase_thresholds[self.current_phase + 1]:self.current_phase += 1task_names = MultiTaskPPOConfig.curriculum_schedule[self.phase_thresholds[self.current_phase]]return [MultiTaskPPOConfig.task_names.index(name) for name in task_names]def collect_experience(self, num_steps, episode):"""集成课程学习和meta controller的经验收集"""current_tasks = self.get_current_tasks(episode)for _ in range(num_steps):# 从当前课程任务中随机选择基础任务base_task_id = np.random.choice(current_tasks)env = self.envs[base_task_id]if not hasattr(env, '_last_obs'):state, _ = env.reset()else:state = env._last_obs# MetaController调整state_tensor = torch.FloatTensor(state).unsqueeze(0).to(MultiTaskPPOConfig.device)with torch.no_grad():task_probs, _ = self.meta_controller(state_tensor)task_probs = task_probs.squeeze().cpu().numpy()# 过滤概率分布mask = np.zeros_like(task_probs)mask[current_tasks] = 1filtered_probs = task_probs * maskfiltered_probs = filtered_probs / (filtered_probs.sum() + 1e-6)# 任务选择策略if np.random.rand() < 0.7:task_id = np.random.choice(current_tasks, p=filtered_probs[current_tasks])else:task_id = np.random.choice(current_tasks)# 记录meta controller决策self.meta_data['states'].append(state_tensor)self.meta_data['chosen_tasks'].append(task_id)# 执行选择的taskenv = self.envs[task_id]with torch.no_grad():task_id_tensor = torch.tensor([task_id], dtype=torch.long, device=MultiTaskPPOConfig.device)action_mean, action_std, value = self.policy(state_tensor, task_id_tensor)dist = Normal(action_mean.float(), action_std.float())  # 确保分布参数是float32action = dist.sample().squeeze(0)log_prob = dist.log_prob(action).sum(-1, keepdim=True)action_np = action.cpu().numpy()next_state, reward, done, trunc, info = env.step(action_np)# 记录数据self.buffer.append({'state': state,'action': action_np,'log_prob': log_prob.cpu(),'reward': float(reward),'done': bool(done),'task_id': task_id,'value': float(value.item()),'success': info.get('success', False)})# 记录meta controller的反馈self.meta_data['rewards'].append(reward)state = next_state if not (done or trunc) else env.reset()[0]def compute_gae(self, values, rewards, dones):"""计算广义优势估计(GAE)"""advantages = []last_advantage = 0next_value = 0next_non_terminal = 1.0for t in reversed(range(len(rewards))):delta = rewards[t] + MultiTaskPPOConfig.gamma * next_value * next_non_terminal - values[t]last_advantage = delta + MultiTaskPPOConfig.gamma * MultiTaskPPOConfig.gae_lambda * next_non_terminal * last_advantageadvantages.append(last_advantage)next_value = values[t]next_non_terminal = 1.0 - dones[t]advantages = torch.tensor(advantages[::-1], dtype=torch.float32).to(MultiTaskPPOConfig.device)returns = advantages + torch.tensor(values, dtype=torch.float32).to(MultiTaskPPOConfig.device)return (advantages - advantages.mean()) / (advantages.std() + 1e-8), returnsdef calculate_task_weights(self):"""基于最近表现计算任务权重"""task_weights = torch.ones(MultiTaskPPOConfig.num_tasks,device=MultiTaskPPOConfig.device)for task_id in range(MultiTaskPPOConfig.num_tasks):if len(self.episode_rewards[task_id]) > 10:# 计算最近10个episode的成功率recent_rewards = list(self.episode_rewards[task_id])[-10:]success_rate = sum(1 for r in recent_rewards if r > 0) / len(recent_rewards)# 动态调整权重if success_rate < 0.3:task_weights[task_id] = 2.0  # 困难任务加倍权重elif success_rate > 0.8:task_weights[task_id] = 0.5  # 简单任务减半权重return task_weights / task_weights.sum()def update_meta_controller(self):"""更新任务选择策略"""if len(self.meta_data['states']) == 0:returnstates = torch.cat(self.meta_data['states'])chosen_tasks = torch.tensor(self.meta_data['chosen_tasks'],device=MultiTaskPPOConfig.device)rewards = torch.tensor(self.meta_data['rewards'],dtype=torch.float32,device=MultiTaskPPOConfig.device)# 清空数据self.meta_data = {'states': [],'chosen_tasks': [],'rewards': []}# 归一化奖励rewards = (rewards - rewards.mean()) / (rewards.std() + 1e-6)# 更新MetaControllertask_probs, logits = self.meta_controller(states)selected_probs = task_probs.gather(1, chosen_tasks.unsqueeze(1))loss = -torch.log(selected_probs + 1e-6) * rewards.unsqueeze(1)loss = loss.mean()self.meta_optimizer.zero_grad()loss.backward()torch.nn.utils.clip_grad_norm_(self.meta_controller.parameters(),MultiTaskPPOConfig.grad_clip)self.meta_optimizer.step()def update_policy(self):"""策略更新方法"""if not self.buffer:return 0, 0, 0# 从缓冲中提取数据batch = list(self.buffer)states = torch.tensor([x['state'] for x in batch],dtype=torch.float32,device=MultiTaskPPOConfig.device)actions = torch.FloatTensor(np.array([x['action'] for x in batch])).to(MultiTaskPPOConfig.device)old_log_probs = torch.cat([x['log_prob'] for x in batch]).to(MultiTaskPPOConfig.device)rewards = torch.FloatTensor([x['reward'] for x in batch]).to(MultiTaskPPOConfig.device)dones = torch.FloatTensor([x['done'] for x in batch]).to(MultiTaskPPOConfig.device)task_ids = torch.tensor([x['task_id'] for x in batch],dtype=torch.long,device=MultiTaskPPOConfig.device)values = torch.FloatTensor([x['value'] for x in batch]).to(MultiTaskPPOConfig.device)successes = torch.FloatTensor([x['success'] for x in batch]).to(MultiTaskPPOConfig.device)# 计算GAE和returnsadvantages, returns = self.compute_gae(values.cpu().numpy(), rewards.cpu().numpy(), dones.cpu().numpy())# 计算任务权重task_weights = self.calculate_task_weights()# 自动混合精度训练total_policy_loss = 0total_value_loss = 0total_entropy = 0for _ in range(MultiTaskPPOConfig.ppo_epochs):# 随机打乱数据perm = torch.randperm(len(batch))for i in range(0, len(batch), MultiTaskPPOConfig.batch_size):idx = perm[i:i + MultiTaskPPOConfig.batch_size]# 获取小批量数据batch_states = states[idx]batch_actions = actions[idx]batch_old_log_probs = old_log_probs[idx]batch_returns = returns[idx]batch_advantages = advantages[idx]batch_task_ids = task_ids[idx]with autocast(device_type=MultiTaskPPOConfig.device.type,enabled=MultiTaskPPOConfig.device.type == 'cuda'):# 前向传播action_means, action_stds, new_values = self.policy(batch_states, batch_task_ids)dist = Normal(action_means, action_stds)new_log_probs = dist.log_prob(batch_actions).sum(-1, keepdim=True)entropy = dist.entropy().mean()# 计算重要性采样比率ratio = (new_log_probs - batch_old_log_probs).exp()# 策略损失surr1 = ratio * batch_advantages.unsqueeze(-1)surr2 = torch.clamp(ratio, 1 - MultiTaskPPOConfig.clip_epsilon,1 + MultiTaskPPOConfig.clip_epsilon) * batch_advantages.unsqueeze(-1)policy_loss_per_task = -torch.min(surr1, surr2)# 应用任务权重selected_weights = task_weights[batch_task_ids].unsqueeze(-1)policy_loss = (policy_loss_per_task * selected_weights).mean()policy_loss -= MultiTaskPPOConfig.entropy_coef * entropy# 值函数损失 (带clip)value_pred_clipped = values[idx] + (new_values - values[idx]).clamp(-MultiTaskPPOConfig.clip_epsilon,MultiTaskPPOConfig.clip_epsilon)value_loss1 = (new_values.squeeze() - batch_returns).pow(2)value_loss2 = (value_pred_clipped.squeeze() - batch_returns).pow(2)value_loss = 0.5 * torch.max(value_loss1, value_loss2).mean()# 总损失loss = policy_loss + value_loss# 反向传播self.scaler.scale(loss).backward()total_policy_loss += policy_loss.item()total_value_loss += value_loss.item()total_entropy += entropy.item()# 梯度裁剪和参数更新self.scaler.unscale_(self.optimizer)torch.nn.utils.clip_grad_norm_(self.policy.shared_net.parameters(), 1.0)torch.nn.utils.clip_grad_norm_(list(self.policy.task_heads.parameters()) +list(self.policy.value_heads.parameters()),0.5)self.scaler.step(self.optimizer)self.scaler.update()self.optimizer.zero_grad()self.scheduler.step()# 衰减动作噪声self.policy.decay_action_std()return (total_policy_loss / MultiTaskPPOConfig.ppo_epochs,total_value_loss / MultiTaskPPOConfig.ppo_epochs,total_entropy / MultiTaskPPOConfig.ppo_epochs)def evaluate_policy(self):"""评估当前策略性能"""eval_results = {i: {'rewards': [], 'successes': []} for i in range(MultiTaskPPOConfig.num_tasks)}for task_id in range(MultiTaskPPOConfig.num_tasks):env = self.envs[task_id]for _ in range(MultiTaskPPOConfig.eval_episodes):state, _ = env.reset()episode_reward = 0done = Falsesuccess = Falsefor _ in range(MultiTaskPPOConfig.max_steps):with torch.no_grad():state_tensor = torch.FloatTensor(state).unsqueeze(0).to(MultiTaskPPOConfig.device)task_id_tensor = torch.tensor([task_id], dtype=torch.long, device=MultiTaskPPOConfig.device)action_mean, _, _ = self.policy(state_tensor, task_id_tensor)action = action_mean.squeeze(0).cpu().numpy()state, reward, done, trunc, info = env.step(action)episode_reward += rewardsuccess = success or info.get('success', False)if done or trunc:breakeval_results[task_id]['rewards'].append(episode_reward)eval_results[task_id]['successes'].append(success)# 记录评估结果for task_id in range(MultiTaskPPOConfig.num_tasks):avg_reward = np.mean(eval_results[task_id]['rewards'])success_rate = np.mean(eval_results[task_id]['successes'])self.eval_rewards[task_id].append(avg_reward)self.eval_success[task_id].append(success_rate)return eval_resultsdef train(self):print(f"开始训练，设备：{MultiTaskPPOConfig.device}")print(f"课程安排：{MultiTaskPPOConfig.curriculum_schedule}")start_time = time.time()# 初始评估self.evaluate_policy()for episode in range(MultiTaskPPOConfig.max_episodes):# 经验收集阶段self.collect_experience(MultiTaskPPOConfig.max_steps, episode)# 策略优化阶段policy_loss, value_loss, entropy = self.update_policy()# MetaController更新self.update_meta_controller()# 记录统计信息for exp in self.buffer:task_id = exp['task_id']self.episode_rewards[task_id].append(exp['reward'])self.episode_lengths[task_id].append(1)# 定期输出日志if (episode + 1) % MultiTaskPPOConfig.log_interval == 0:avg_rewards = {k: np.mean(v) if v else 0 for k, v in self.episode_rewards.items()}success_rates = {k: np.mean([1 if r > 0 else 0 for r in v]) if v else 0for k, v in self.episode_rewards.items()}time_cost = time.time() - start_time# 打印当前课程阶段current_task_names = MultiTaskPPOConfig.curriculum_schedule[self.phase_thresholds[self.current_phase]]print(f"\nEpisode {episode + 1:5d} | Time: {time_cost:6.1f}s")print(f"当前课程阶段: {current_task_names} (Phase {self.current_phase})")print(f"动作标准差: {self.policy.current_std:.3f} | 学习率: {self.scheduler.get_last_lr()[0]:.2e}")for task_id in range(MultiTaskPPOConfig.num_tasks):task_name = MultiTaskPPOConfig.task_names[task_id]print(f"  {task_name:25s} | Avg Reward: {avg_rewards[task_id]:7.2f} | Success Rate: {success_rates[task_id]:.2f}")print(f"  Policy Loss: {policy_loss:.4f} | Value Loss: {value_loss:.4f} | Entropy: {entropy:.4f}")start_time = time.time()# 定期评估if (episode + 1) % MultiTaskPPOConfig.eval_interval == 0:eval_results = self.evaluate_policy()if (episode + 1) % 1000 == 0:print("\n评估结果:")for task_id in range(MultiTaskPPOConfig.num_tasks):task_name = MultiTaskPPOConfig.task_names[task_id]avg_reward = np.mean(eval_results[task_id]['rewards'])success_rate = np.mean(eval_results[task_id]['successes'])print(f"  {task_name:25s} | Avg Reward: {avg_reward:7.2f} | Success Rate: {success_rate:.2f}")# 训练结束保存模型torch.save({'policy_state_dict': self.policy.state_dict(),'meta_controller_state_dict': self.meta_controller.state_dict(),'optimizer_state_dict': self.optimizer.state_dict()}, "multitask_ppo_model.pth")if __name__ == "__main__":trainer = EnhancedMultiTaskPPOTrainer()print(f"状态维度: {trainer.state_dim}, 动作维度: {trainer.action_dim}")trainer.train()

部分输出为：

Episode    50 | Time:  216.6s
当前课程阶段: ['reach-v2-goal-observable'] (Phase 0)
动作标准差: 1.427 | 学习率: 5.00e-05reach-v2-goal-observable  | Avg Reward:    1.42 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.00 | Success Rate: 0.00pick-place-v2-goal-observable | Avg Reward:    0.00 | Success Rate: 0.00Policy Loss: -0.1777 | Value Loss: 471.4303 | Entropy: 1.7773Episode   100 | Time:  193.3s
当前课程阶段: ['reach-v2-goal-observable'] (Phase 0)
动作标准差: 1.357 | 学习率: 5.00e-05reach-v2-goal-observable  | Avg Reward:    1.42 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.00 | Success Rate: 0.00pick-place-v2-goal-observable | Avg Reward:    0.00 | Success Rate: 0.00Policy Loss: -0.1729 | Value Loss: 357.7264 | Entropy: 1.7293......Episode  2800 | Time:  198.6s
当前课程阶段: ['reach-v2-goal-observable', 'push-v2-goal-observable'] (Phase 1)
动作标准差: 0.200 | 学习率: 4.11e-05reach-v2-goal-observable  | Avg Reward:    1.44 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.00 | Success Rate: 0.00Policy Loss: 0.0092 | Value Loss: 191.3147 | Entropy: -0.0918Episode  2850 | Time:  212.2s
当前课程阶段: ['reach-v2-goal-observable', 'push-v2-goal-observable'] (Phase 1)
动作标准差: 0.200 | 学习率: 4.08e-05reach-v2-goal-observable  | Avg Reward:    1.44 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.00 | Success Rate: 0.00Policy Loss: 0.0090 | Value Loss: 183.6324 | Entropy: -0.0902Episode  2900 | Time:  210.4s
当前课程阶段: ['reach-v2-goal-observable', 'push-v2-goal-observable'] (Phase 1)
动作标准差: 0.200 | 学习率: 4.05e-05reach-v2-goal-observable  | Avg Reward:    1.44 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.00 | Success Rate: 0.00Policy Loss: 0.0089 | Value Loss: 188.5185 | Entropy: -0.0889Episode  2950 | Time:  210.1s
当前课程阶段: ['reach-v2-goal-observable', 'push-v2-goal-observable'] (Phase 1)
动作标准差: 0.200 | 学习率: 4.02e-05reach-v2-goal-observable  | Avg Reward:    1.44 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.00 | Success Rate: 0.00Policy Loss: 0.0087 | Value Loss: 183.0386 | Entropy: -0.0874Episode  3000 | Time:  212.0s
当前课程阶段: ['reach-v2-goal-observable', 'push-v2-goal-observable'] (Phase 1)
动作标准差: 0.200 | 学习率: 3.99e-05reach-v2-goal-observable  | Avg Reward:    1.45 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.00 | Success Rate: 0.00Policy Loss: 0.0086 | Value Loss: 182.9761 | Entropy: -0.0858评估结果:reach-v2-goal-observable  | Avg Reward:  106.66 | Success Rate: 0.00push-v2-goal-observable   | Avg Reward:    3.99 | Success Rate: 0.00pick-place-v2-goal-observable | Avg Reward:    4.49 | Success Rate: 0.00Episode  3050 | Time:  234.3s
当前课程阶段: ['push-v2-goal-observable', 'pick-place-v2-goal-observable'] (Phase 2)
动作标准差: 0.200 | 学习率: 3.96e-05reach-v2-goal-observable  | Avg Reward:    1.45 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.02 | Success Rate: 1.00Policy Loss: 0.0084 | Value Loss: 28.1028 | Entropy: -0.0843Episode  3100 | Time:  210.3s
当前课程阶段: ['push-v2-goal-observable', 'pick-place-v2-goal-observable'] (Phase 2)
动作标准差: 0.200 | 学习率: 3.93e-05reach-v2-goal-observable  | Avg Reward:    1.45 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.02 | Success Rate: 1.00Policy Loss: 0.0083 | Value Loss: 0.1660 | Entropy: -0.0829Episode  3150 | Time:  209.8s
当前课程阶段: ['push-v2-goal-observable', 'pick-place-v2-goal-observable'] (Phase 2)
动作标准差: 0.200 | 学习率: 3.90e-05reach-v2-goal-observable  | Avg Reward:    1.45 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.02 | Success Rate: 1.00Policy Loss: 0.0082 | Value Loss: 0.1506 | Entropy: -0.0818Episode  3200 | Time:  210.2s
当前课程阶段: ['push-v2-goal-observable', 'pick-place-v2-goal-observable'] (Phase 2)
动作标准差: 0.200 | 学习率: 3.86e-05reach-v2-goal-observable  | Avg Reward:    1.45 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.02 | Success Rate: 1.00Policy Loss: 0.0080 | Value Loss: 0.1429 | Entropy: -0.0801Episode  3250 | Time:  210.3s
当前课程阶段: ['push-v2-goal-observable', 'pick-place-v2-goal-observable'] (Phase 2)
动作标准差: 0.200 | 学习率: 3.83e-05reach-v2-goal-observable  | Avg Reward:    1.45 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.02 | Success Rate: 1.00Policy Loss: 0.0079 | Value Loss: 0.1725 | Entropy: -0.0785Episode  3300 | Time:  209.7s
当前课程阶段: ['push-v2-goal-observable', 'pick-place-v2-goal-observable'] (Phase 2)
动作标准差: 0.200 | 学习率: 3.80e-05reach-v2-goal-observable  | Avg Reward:    1.45 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.02 | Success Rate: 1.00Policy Loss: 0.0077 | Value Loss: 0.1990 | Entropy: -0.0771Episode  3350 | Time:  209.5s
当前课程阶段: ['push-v2-goal-observable', 'pick-place-v2-goal-observable'] (Phase 2)
动作标准差: 0.200 | 学习率: 3.76e-05reach-v2-goal-observable  | Avg Reward:    1.45 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.02 | Success Rate: 1.00Policy Loss: 0.0076 | Value Loss: 0.2084 | Entropy: -0.0758Episode  3400 | Time:  210.1s
当前课程阶段: ['push-v2-goal-observable', 'pick-place-v2-goal-observable'] (Phase 2)
动作标准差: 0.200 | 学习率: 3.73e-05reach-v2-goal-observable  | Avg Reward:    1.45 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.02 | Success Rate: 1.00Policy Loss: 0.0075 | Value Loss: 0.2057 | Entropy: -0.0745Episode  3450 | Time:  210.9s
当前课程阶段: ['push-v2-goal-observable', 'pick-place-v2-goal-observable'] (Phase 2)
动作标准差: 0.200 | 学习率: 3.70e-05reach-v2-goal-observable  | Avg Reward:    1.45 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.02 | Success Rate: 1.00Policy Loss: 0.0073 | Value Loss: 0.2251 | Entropy: -0.0733Episode  3500 | Time:  210.1s
当前课程阶段: ['push-v2-goal-observable', 'pick-place-v2-goal-observable'] (Phase 2)
动作标准差: 0.200 | 学习率: 3.66e-05reach-v2-goal-observable  | Avg Reward:    1.45 | Success Rate: 1.00push-v2-goal-observable   | Avg Reward:    0.05 | Success Rate: 1.00pick-place-v2-goal-observable | Avg Reward:    0.02 | Success Rate: 1.00Policy Loss: 0.0072 | Value Loss: 0.2199 | Entropy: -0.0723......

本文来自互联网用户投稿，该文观点仅代表作者本人，不代表本站立场。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如若转载，请注明出处：http://www.mzph.cn/web/77805.shtml

如若内容造成侵权/违法违规/事实不符，请联系多彩编程网进行投诉反馈email:809451989@qq.com，一经查实，立即删除！

相关文章

前端——CSS1

前端——CSS1

一&#xff0c;概述 CSS&#xff08;Cascading Style Sheets&#xff09;&#xff08;级联样式表&#xff09; css是一种样式表语言&#xff0c;为html标签修饰定义外观&#xff0c;分工不同 涉及&#xff1a;对网页的文字、背景、宽、高、布局进行修饰 分为内嵌样式表&…
阅读更多...
赋能航天教育：高校卫星仿真教学实验平台解决方案

赋能航天教育：高校卫星仿真教学实验平台解决方案

​​​​​​ 随着全球航天事业的飞速发展&#xff0c;对高素质航天人才的需求日益增长。如何在高校阶段提前锻炼学生的航天工程实践能力&#xff0c;成为教育界的重要命题。作为领先的通信与网络技术供应商&#xff0c;IPLOOK基于自身在5G核心网、卫星通信及仿真平台领域的深…
阅读更多...
Python爬虫（10）Python数据存储实战：基于pymongo的MongoDB开发深度指南

Python爬虫（10）Python数据存储实战：基于pymongo的MongoDB开发深度指南

目录 一、为什么需要文档型数据库&#xff1f;1.1 数据存储的范式变革1.2 pymongo的核心优势 二、pymongo核心操作全解析2.1 环境准备2.2 数据库连接与CRUD操作2.3 聚合管道实战2.4 分批次插入百万级数据&#xff08;进阶&#xff09;2.5 分批次插入百万级数据&#xff08;进阶…
阅读更多...
Springboot 手搓 后端 滑块验证码生成

Springboot 手搓 后端 滑块验证码生成

目录 一、效果演示 二、后端滑块验证码生成思路 三、原理解析 四、核心代码拿走 滑块验证码react前端实现&#xff0c;见我的这篇博客&#xff1a;前端 React 弹窗式 滑动验证码实现_react中使用阿里云滑块验证码2.0前端接入及相关视觉-CSDN博客 一、效果演示 生成的案例…
阅读更多...
关于flink两阶段提交高并发下程序卡住问题

关于flink两阶段提交高并发下程序卡住问题

先抛出代码 package com.dpf.flink;import com.dpf.flink.sink.MysqlSink; import org.apache.flink.api.common.serialization.SimpleStringSchema; import org.apache.flink.api.common.typeinfo.Types; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.…
阅读更多...
html css js网页制作成品——HTML+CSS+js美甲店网页设计（5页）附源码

html css js网页制作成品——HTML+CSS+js美甲店网页设计（5页）附源码

美甲店 目录 一、&#x1f468;‍&#x1f393;网站题目 二、✍️网站描述 三、&#x1f4da;网站介绍 四、&#x1f310;网站效果 五、&#x1fa93; 代码实现 &#x1f9f1;HTML 六、&#x1f947; 如何让学习不再盲目 七、&#x1f381;更多干货 一、&#x1f468;‍&a…
阅读更多...
LeetCode[347]前K个高频元素

LeetCode[347]前K个高频元素

思路&#xff1a; 使用小顶堆&#xff0c;最小的元素都出去了&#xff0c;省的就是大&#xff0c;高频的元素了&#xff0c;所以要维护一个小顶堆&#xff0c;使用map存元素高频变化&#xff0c;map存堆里&#xff0c;然后输出堆的东西就行了 代码&#xff1a; class Solution…
阅读更多...
2024年网站开发语言选择指南：PHP/Java/Node.js/Python如何选型？

2024年网站开发语言选择指南：PHP/Java/Node.js/Python如何选型？

2024年网站开发语言选择指南&#xff1a;PHP/Java/Node.js/Python如何选型&#xff1f; 一、8大主流Web开发语言技术对比 1. PHP开发&#xff1a;中小型网站的首选方案 最新版本&#xff1a;PHP 8.3&#xff08;2023年11月发布&#xff09;核心优势&#xff1a; 全球78%的网站…
阅读更多...
从数据结构说起(一)

从数据结构说起(一)

1 揭开数据结构神奇的面纱 1.1 初识数据结构 在C的标准库模板&#xff08;Standard Template Library,STL&#xff09;课程上&#xff0c;我初次结识了《数据结构》。C语言提供的标准库模板是面向对象程序设计与泛型程序设计思想相结合的典范。所谓的泛型编程就是编写不依赖于具…
阅读更多...
JAVA--- 关键字static

JAVA--- 关键字static

之前我们学习了JAVA 面向对象的一些基本知识&#xff0c;今天来进阶一下&#xff01;&#xff01;&#xff01; static关键字 static表示静态&#xff0c;是JAVA中的一个修饰符&#xff0c;可以修饰成员方法&#xff0c;成员变量&#xff0c;可用于修饰类的成员&#xff08;变…
阅读更多...
4.27比赛总结

4.27比赛总结

文章目录 T1T2法一&#xff1a;倍增求 LCA法二&#xff1a;Dijkstra 求最短路法三&#xff1a;dfs 求深度 T3T4总结 T1 一道非常简单的题&#xff0c;结果我因为一句话没写挂了 80pts…… 题目中没写 a a a 数组要按照 b b b 数组的顺序&#xff0c;所以对于最大方案&#x…
阅读更多...
数据一致性巡检总结：基于分桶采样的设计与实现

数据一致性巡检总结：基于分桶采样的设计与实现

数据一致性巡检总结&#xff1a;基于分桶采样的设计与实现 背景 在分布式系统中&#xff0c;缓存&#xff08;如 Redis&#xff09;与数据库&#xff08;如 MySQL&#xff09;之间的数据一致性问题是一个常见的挑战。由于缓存的引入&#xff0c;数据在缓存和数据库之间可能存…
阅读更多...
SpringBoot与Druid整合，实现主从数据库同步

SpringBoot与Druid整合，实现主从数据库同步

通过引入主从数据库同步系统&#xff0c;可以显著提升平台的性能和稳定性&#xff0c;同时保证数据的一致性和安全性。Druid连接池也提供了强大的监控和安全防护功能&#xff0c;使得整个系统更加健壮和可靠。 我们为什么选择Druid&#xff1f; 高效的连接管理&#xff1a;Dru…
阅读更多...
在Linux系统中安装MySQL，二进制包版

在Linux系统中安装MySQL，二进制包版

1、检查是否已安装数据库&#xff08;rpm软件包管理器&#xff09; rpm -qa | grep mysql rpm -qa | grep mariadb #centOS7自带mariadb与mysql数据库冲突2、删除已有数据库 rpm -e –nodeps 软件名称 3、官网下载MySQL包 4、上传 # 使用FinalShell或Xshell工具上传&#…
阅读更多...
【含文档+PPT+源码】基于SpringBoot电脑DIY装机教程网站的设计与实现

【含文档+PPT+源码】基于SpringBoot电脑DIY装机教程网站的设计与实现

项目介绍 本课程演示的是一款 基于SpringBoot电脑DIY装机教程网站的设计与实现&#xff0c;主要针对计算机相关专业的正在做毕设的学生与需要项目实战练习的 Java 学习者。 1.包含&#xff1a;项目源码、项目文档、数据库脚本、软件工具等所有资料 2.带你从零开始部署运行本套…
阅读更多...
Spring Boot 缓存机制：从原理到实践

Spring Boot 缓存机制：从原理到实践

文章目录 一、引言二、Spring Boot 缓存机制原理2.1 缓存抽象层2.2 缓存注解2.3 缓存管理器 三、入门使用3.1 引入依赖3.2 配置缓存3.3 启用缓存3.4 使用缓存注解3.5 实体类 四、踩坑记录4.1 缓存键生成问题4.2 缓存过期与更新问题4.3 事务与缓存的一致性问题 五、心得体会5.1 …
阅读更多...
Spark读取Apollo配置

Spark读取Apollo配置

--conf spark.driver.extraJavaOptions-Dapp.idapollo的app.id -Denvfat -Dapollo.clusterfat -Dfat_metaapollo的meta地址 --conf spark.executor.extraJavaOptions-Dapp.idapollo的app.id -Denvfat -Dapollo.clusterfat -Dfat_metaapollo的meta地址 在spark的提交命令中&…
阅读更多...
[逆向工程]如何理解小端序？逆向工程中的字节序陷阱与实战解析

[逆向工程]如何理解小端序？逆向工程中的字节序陷阱与实战解析

[逆向工程]如何理解小端序&#xff1f;逆向工程中的字节序陷阱与实战解析 关键词&#xff1a;逆向工程、小端序、字节序、二进制分析、数据解析 引言&#xff1a;为什么字节序是逆向工程师的必修课&#xff1f; 在逆向工程中&#xff0c;分析二进制数据是最基础的任务之一。…
阅读更多...
项目三 - 任务2：创建笔记本电脑类（一爹多叔）

项目三 - 任务2：创建笔记本电脑类（一爹多叔）

在本次实战中&#xff0c;我们通过Java的单根继承和多接口实现特性&#xff0c;设计了一个笔记本电脑类。首先创建了Computer抽象类&#xff0c;提供计算的抽象方法&#xff0c;模拟电脑的基本功能。接着定义了NetCard和USB两个接口&#xff0c;分别包含连接网络和USB设备的抽象…
阅读更多...
ElasticSearch深入解析（六）：集群核心配置

ElasticSearch深入解析（六）：集群核心配置

1.开发模式和生产模式 Elasticsearch默认运行在开发模式下&#xff0c;此模式允许节点在配置存在错误时照常启动&#xff0c;仅将警告信息写入日志文件。而生产模式则更为严格&#xff0c;一旦检测到配置错误&#xff0c;节点将无法启动&#xff0c;这是一种保障系统稳定性的安…
阅读更多...
最新文章

Copyright @ 2022~2023