Transformer 与 LSTM 在时序回归中的实践与优化

🧠 深度学习混合模型：Transformer 与 LSTM 在时序回归中的实践与优化

在处理多特征输入、多目标输出的时序回归任务时，结合 Transformer 和 LSTM 的混合模型已成为一种有效的解决方案。Transformer 擅长捕捉长距离依赖关系，而 LSTM 在处理序列数据时表现出色。通过将两者结合，可以充分发挥各自的优势，提高模型的预测性能。

📊 数据生成与预处理

首先，我们生成一个包含多个特征的时序数据集，并进行必要的预处理。

import numpy as np
import pandas as pd
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split# 设置随机种子以确保结果可复现
np.random.seed(42)# 生成时间序列数据
n_samples = 1000
time_steps = 10
n_features = 5
X = np.random.rand(n_samples, time_steps, n_features)
y = np.random.rand(n_samples, 1)  # 假设我们有一个目标变量# 数据归一化
scaler_X = MinMaxScaler()
scaler_y = MinMaxScaler()X_scaled = X.reshape(-1, n_features)
X_scaled = scaler_X.fit_transform(X_scaled)
X_scaled = X_scaled.reshape(n_samples, time_steps, n_features)y_scaled = scaler_y.fit_transform(y)# 划分训练集和测试集
X_train, X_test, y_train, y_test = train_test_split(X_scaled, y_scaled, test_size=0.2, random_state=42)

🧩 模型架构设计

我们设计一个结合 Transformer 和 LSTM 的混合模型架构。

import tensorflow as tf
from tensorflow.keras import layers, modelsdef build_transformer_lstm_model(input_shape, lstm_units=64, transformer_units=64, num_heads=4, num_layers=2, dropout_rate=0.1):inputs = layers.Input(shape=input_shape)# LSTM 层x = layers.LSTM(lstm_units, return_sequences=True)(inputs)x = layers.Dropout(dropout_rate)(x)# Transformer 层for _ in range(num_layers):attention = layers.MultiHeadAttention(num_heads=num_heads, key_dim=transformer_units)(x, x)x = layers.Add()([x, attention])x = layers.LayerNormalization()(x)x = layers.Dropout(dropout_rate)(x)# 输出层x = layers.GlobalAveragePooling1D()(x)x = layers.Dense(64, activation='relu')(x)x = layers.Dropout(dropout_rate)(x)outputs = layers.Dense(1)(x)model = models.Model(inputs, outputs)return model# 构建模型
input_shape = (X_train.shape[1], X_train.shape[2])
model = build_transformer_lstm_model(input_shape)
model.compile(optimizer='adam', loss='mean_squared_error', metrics=['mae'])

🏋️‍♂️ 模型训练与评估

from tensorflow.keras.callbacks import EarlyStopping# 定义早停机制
early_stopping = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)# 训练模型
history = model.fit(X_train, y_train, epochs=50, batch_size=32, validation_data=(X_test, y_test), callbacks=[early_stopping])# 评估模型
loss, mae = model.evaluate(X_test, y_test)
print(f"Test Loss: {loss}, Test MAE: {mae}")

🔧 超参数调优

我们使用 Keras Tuner 进行超参数调优。

import keras_tuner as ktdef model_builder(hp):model = build_transformer_lstm_model(input_shape)model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=hp.Float('learning_rate', min_value=1e-5, max_value=1e-2, sampling='log')),loss='mean_squared_error',metrics=['mae'])return model# 定义调优器
tuner = kt.Hyperband(model_builder,objective='val_loss',max_epochs=10,factor=3,directory='hyperband',project_name='transformer_lstm'
)# 执行超参数调优
tuner.search(X_train, y_train, epochs=50, validation_data=(X_test, y_test), callbacks=[early_stopping])# 获取最佳超参数
best_hps = tuner.get_best_hyperparameters()[0]
print(f"Best learning rate: {best_hps.get('learning_rate')}")

📈 结果可视化

import matplotlib.pyplot as plt# 绘制训练过程中的损失和 MAE
plt.figure(figsize=(12, 6))
plt.subplot(1, 2, 1)
plt.plot(history.history['loss'], label='Train Loss')
plt.plot(history.history['val_loss'], label='Val Loss')
plt.title('Loss Over Epochs')
plt.legend()plt.subplot(1, 2, 2)
plt.plot(history.history['mae'], label='Train MAE')
plt.plot(history.history['val_mae'], label='Val MAE')
plt.title('MAE Over Epochs')
plt.legend()plt.tight_layout()
plt.show()