1¶
In [ ]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import os
import warnings
warnings.filterwarnings('ignore')
from google.colab import drive
drive.mount('/content/drive')
os.chdir('/content/drive/MyDrive/690/MiniProject2')
Mounted at /content/drive
In [ ]:
# load datasets
news_df = pd.read_csv('news.csv')
price_df = pd.read_csv('price.csv')
print(f"News: {news_df.shape[0]} articles, {news_df['tickers'].nunique()} unique tickers")
print(f"Price: {price_df.shape[0]} rows, {price_df['ticker'].nunique()} unique tickers")
News: 20550 articles, 381 unique tickers Price: 603840 rows, 480 unique tickers
In [ ]:
news_df.head()
Out[Â ]:
| publication_datetime | title | body | tickers | |
|---|---|---|---|---|
| 0 | 2017-01-03 | World News: Police Question Netanyahu Over Gifts | "We pay attention to publications in the media... | EL |
| 1 | 2017-01-03 | Business News: Nestle Turns to New CEO for Hea... | Nestle, the world's largest packaged-food comp... | GIS |
| 2 | 2017-01-03 | Business News: Vermont Drug Law Faces Limits -... | The Vermont law, enacted in June, instructed s... | ABBV |
| 3 | 2017-01-03 | Life & Arts -- Travel: How Hotel Companies Lau... | Travelers are about to see a flurry of new hot... | HLT |
| 4 | 2017-01-03 | Businesses Ready to Ramp Up Investment --- Aft... | The Federal Reserve last month signaled intere... | HD |
In [ ]:
price_df.head()
Out[Â ]:
| Date | ticker | close | |
|---|---|---|---|
| 0 | 2017-01-03 | A | 43.743862 |
| 1 | 2017-01-04 | A | 44.317825 |
| 2 | 2017-01-05 | A | 43.790924 |
| 3 | 2017-01-06 | A | 45.155281 |
| 4 | 2017-01-09 | A | 45.296406 |
In [ ]:
# date ranges
news_df['publication_datetime'] = pd.to_datetime(news_df['publication_datetime'])
price_df['Date'] = pd.to_datetime(price_df['Date'])
print(f"News date range: {news_df['publication_datetime'].min().date()} to {news_df['publication_datetime'].max().date()}")
print(f"Price date range: {price_df['Date'].min().date()} to {price_df['Date'].max().date()}")
News date range: 2017-01-03 to 2020-12-30 Price date range: 2017-01-03 to 2021-12-30
In [ ]:
# monthly article distribution
monthly_count = news_df.resample('M', on='publication_datetime').size()
plt.figure(figsize=(12, 5))
monthly_count.plot(kind='bar', color='skyblue')
plt.title('Number of News Articles per Month')
plt.xlabel('Month')
plt.ylabel('Count')
plt.xticks(ticks=range(len(monthly_count)), labels=monthly_count.index.strftime('%Y-%m'), rotation=45, ha='right')
plt.grid(axis='y')
plt.tight_layout()
plt.show()
2¶
In [ ]:
# compute daily returns
price_df['daily_return'] = price_df.groupby('ticker')['close'].transform(lambda x: x.pct_change())
price_df = price_df.dropna()
# time-series split
train_mask = news_df['publication_datetime'] < '2019-01-01'
val_mask = (news_df['publication_datetime'] >= '2019-01-01') & (news_df['publication_datetime'] < '2020-01-01')
test_mask = news_df['publication_datetime'] >= '2020-01-01'
print(f"Train: {train_mask.sum()} articles (before 2019)")
print(f"Val: {val_mask.sum()} articles (2019)")
print(f"Test: {test_mask.sum()} articles (2020+)")
Train: 10320 articles (before 2019) Val: 5338 articles (2019) Test: 4892 articles (2020+)
In [ ]:
# visualize the split
fig, ax = plt.subplots(figsize=(12, 4))
for label, mask, color in [('Train', train_mask, 'steelblue'), ('Validation', val_mask, 'orange'), ('Test', test_mask, 'green')]:
subset = news_df[mask].resample('M', on='publication_datetime').size()
ax.bar(subset.index, subset.values, width=25, label=label, color=color, alpha=0.7)
ax.set_title('Articles per Month by Split')
ax.set_ylabel('Count')
ax.legend()
plt.tight_layout()
plt.show()
3¶
3.1¶
In [ ]:
import torch
from transformers import BertTokenizer, BertForSequenceClassification
# device for FinBERT
fb_device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print(f"FinBERT device: {fb_device}")
# load FinBERT
finbert_tokenizer = BertTokenizer.from_pretrained('yiyanghkust/finbert-tone')
finbert = BertForSequenceClassification.from_pretrained('yiyanghkust/finbert-tone')
finbert.to(fb_device)
finbert.eval()
print(f"Max input tokens: {finbert.config.max_position_embeddings}")
FinBERT device: cuda
vocab.txt: 0.00B [00:00, ?B/s]
Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. WARNING:huggingface_hub.utils._http:Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads.
config.json: 0%| | 0.00/533 [00:00<?, ?B/s]
pytorch_model.bin: 0%| | 0.00/439M [00:00<?, ?B/s]
Loading weights: 0%| | 0/201 [00:00<?, ?it/s]
BertForSequenceClassification LOAD REPORT from: yiyanghkust/finbert-tone Key | Status | | -----------------------------+------------+--+- bert.embeddings.position_ids | UNEXPECTED | | Notes: - UNEXPECTED :can be ignored when loading from different task/architecture; not ok if you expect identical arch.
Max input tokens: 512
In [ ]:
def predict_sentiment_batch(texts, batch_size=16):
"""Run FinBERT on a list of texts in batches."""
all_scores = []
for i in range(0, len(texts), batch_size):
batch = texts[i:i+batch_size]
inputs = finbert_tokenizer(batch, return_tensors='pt', truncation=True,
padding=True, max_length=512)
inputs = {k: v.to(fb_device) for k, v in inputs.items()}
with torch.no_grad():
outputs = finbert(**inputs)
scores = outputs.logits.softmax(dim=1).cpu().numpy()
all_scores.append(scores)
if (i // batch_size) % 200 == 0:
print(f" Processed {i}/{len(texts)} articles...")
return np.vstack(all_scores)
In [ ]:
# check if cached results exist, otherwise run FinBERT
if os.path.exists('news_w_sentiment.csv'):
print("Loading cached FinBERT sentiment from news_w_sentiment.csv")
news_w_sent = pd.read_csv('news_w_sentiment.csv')
news_w_sent['publication_datetime'] = pd.to_datetime(news_w_sent['publication_datetime'])
else:
print("Running FinBERT sentiment analysis...")
texts = news_df['body'].fillna('').tolist()
scores = predict_sentiment_batch(texts, batch_size=16)
news_df['neutral'] = scores[:, 0]
news_df['positive'] = scores[:, 1]
news_df['negative'] = scores[:, 2]
news_df.to_csv('news_w_sentiment.csv', index=False)
news_w_sent = news_df.copy()
print("Done. Saved to news_w_sentiment.csv")
Running FinBERT sentiment analysis... Processed 0/20550 articles... Processed 3200/20550 articles...
In [ ]:
# sentiment score = positive - negative
news_w_sent['sentiment'] = news_w_sent['positive'] - news_w_sent['negative']
# distribution
fig, axes = plt.subplots(1, 3, figsize=(15, 4))
for ax, col in zip(axes, ['neutral', 'positive', 'negative']):
ax.hist(news_w_sent[col], bins=50, alpha=0.7, edgecolor='black')
ax.set_title(f'{col.capitalize()} Score')
ax.set_xlabel('Probability')
plt.suptitle('FinBERT Sentiment Score Distributions')
plt.tight_layout()
plt.show()
print(f"Mean sentiment: {news_w_sent['sentiment'].mean():.4f}")
print(f"Std sentiment: {news_w_sent['sentiment'].std():.4f}")
Mean sentiment: 0.0546 Std sentiment: 0.5147
3.2¶
In [ ]:
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
from transformers import BertTokenizer, BertModel
# device setup - supports CUDA, MPS (Apple Silicon), or CPU
if torch.cuda.is_available():
device = torch.device('cuda')
elif hasattr(torch.backends, 'mps') and torch.backends.mps.is_available():
device = torch.device('mps')
else:
device = torch.device('cpu')
print(f"Using device: {device}")
Using device: cuda
In [ ]:
# load BERT for feature extraction
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
bert_model = BertModel.from_pretrained('bert-base-uncased')
bert_model.eval()
bert_model.to(device)
print("BERT model loaded.")
tokenizer_config.json: 0%| | 0.00/48.0 [00:00<?, ?B/s]
vocab.txt: 0.00B [00:00, ?B/s]
tokenizer.json: 0.00B [00:00, ?B/s]
config.json: 0%| | 0.00/570 [00:00<?, ?B/s]
model.safetensors: 0%| | 0.00/440M [00:00<?, ?B/s]
Loading weights: 0%| | 0/199 [00:00<?, ?it/s]
BertModel LOAD REPORT from: bert-base-uncased Key | Status | | -------------------------------------------+------------+--+- cls.predictions.bias | UNEXPECTED | | cls.predictions.transform.LayerNorm.bias | UNEXPECTED | | cls.predictions.transform.LayerNorm.weight | UNEXPECTED | | cls.predictions.transform.dense.bias | UNEXPECTED | | cls.seq_relationship.bias | UNEXPECTED | | cls.predictions.transform.dense.weight | UNEXPECTED | | cls.seq_relationship.weight | UNEXPECTED | | Notes: - UNEXPECTED :can be ignored when loading from different task/architecture; not ok if you expect identical arch.
BERT model loaded.
In [ ]:
class NewsDataset(Dataset):
def __init__(self, news_data, price_data, tokenizer, bert_model, device, max_length=512, cache_path=None):
self.news_data = news_data.reset_index(drop=True)
self.price_data = price_data
self.return_map = price_data.set_index(['Date', 'ticker'])['daily_return'].to_dict()
# compute or load cached embeddings
if cache_path and os.path.exists(cache_path):
print(f"Loading cached embeddings from {cache_path}")
self.embeddings = np.load(cache_path)
else:
print(f"Computing BERT embeddings for {len(news_data)} articles...")
self.embeddings = self._compute_embeddings(tokenizer, bert_model, device, max_length)
if cache_path:
np.save(cache_path, self.embeddings)
print(f"Saved embeddings to {cache_path}")
# precompute labels (next trading day return)
self.labels = self._compute_labels()
def _compute_embeddings(self, tokenizer, bert_model, device, max_length):
embeddings = []
with torch.no_grad():
for idx in range(len(self.news_data)):
if idx % 2000 == 0:
print(f" {idx}/{len(self.news_data)}")
text = str(self.news_data.iloc[idx]['body'])
inputs = tokenizer(text, return_tensors='pt', max_length=max_length,
padding='max_length', truncation=True)
inputs = {k: v.to(device) for k, v in inputs.items()}
outputs = bert_model(**inputs)
emb = outputs.last_hidden_state[:, 0, :].squeeze().cpu().numpy()
embeddings.append(emb)
return np.array(embeddings)
def _compute_labels(self):
labels = []
for idx in range(len(self.news_data)):
row = self.news_data.iloc[idx]
date = row['publication_datetime']
ticker = row['tickers']
# find next trading day for this ticker
future = self.price_data[
(self.price_data['Date'] > date) & (self.price_data['ticker'] == ticker)
]['Date']
if len(future) > 0:
next_date = future.iloc[0]
labels.append(self.return_map.get((next_date, ticker), 0.0))
else:
labels.append(0.0)
return np.array(labels, dtype=np.float32)
def __len__(self):
return len(self.news_data)
def __getitem__(self, idx):
return torch.FloatTensor(self.embeddings[idx]), torch.FloatTensor([self.labels[idx]])
In [ ]:
class ReturnPredictor(nn.Module):
def __init__(self, input_size=768, hidden_size=256, dropout=0.2):
super().__init__()
self.network = nn.Sequential(
nn.Linear(input_size, hidden_size),
nn.ReLU(),
nn.Dropout(dropout),
nn.Linear(hidden_size, hidden_size // 2),
nn.ReLU(),
nn.Dropout(dropout),
nn.Linear(hidden_size // 2, 1)
)
def forward(self, x):
return self.network(x)
In [ ]:
torch.manual_seed(42)
print("Creating datasets (this may take a while on first run)...")
train_dataset = NewsDataset(news_df[train_mask], price_df, tokenizer, bert_model, device,
cache_path='train_embeddings.npy')
val_dataset = NewsDataset(news_df[val_mask], price_df, tokenizer, bert_model, device,
cache_path='val_embeddings.npy')
test_dataset = NewsDataset(news_df[test_mask], price_df, tokenizer, bert_model, device,
cache_path='test_embeddings.npy')
print(f"\nDataset sizes - Train: {len(train_dataset)}, Val: {len(val_dataset)}, Test: {len(test_dataset)}")
Creating datasets (this may take a while on first run)... Computing BERT embeddings for 10320 articles... 0/10320 2000/10320 4000/10320 6000/10320 8000/10320 10000/10320 Saved embeddings to train_embeddings.npy Computing BERT embeddings for 5338 articles... 0/5338 2000/5338 4000/5338 Saved embeddings to val_embeddings.npy Computing BERT embeddings for 4892 articles... 0/4892 2000/4892 4000/4892 Saved embeddings to test_embeddings.npy Dataset sizes - Train: 10320, Val: 5338, Test: 4892
In [ ]:
# train the baseline model
train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=32)
test_loader = DataLoader(test_dataset, batch_size=32)
model = ReturnPredictor().to(device)
criterion = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
num_epochs = 10
train_losses, val_losses = [], []
for epoch in range(num_epochs):
model.train()
epoch_loss = 0
for X, y in train_loader:
X, y = X.to(device), y.to(device)
optimizer.zero_grad()
loss = criterion(model(X), y)
loss.backward()
optimizer.step()
epoch_loss += loss.item()
model.eval()
v_loss = 0
with torch.no_grad():
for X, y in val_loader:
X, y = X.to(device), y.to(device)
v_loss += criterion(model(X), y).item()
train_losses.append(epoch_loss / len(train_loader))
val_losses.append(v_loss / len(val_loader))
print(f"Epoch {epoch+1}/{num_epochs} - Train Loss: {train_losses[-1]:.6f}, Val Loss: {val_losses[-1]:.6f}")
Epoch 1/10 - Train Loss: 0.000751, Val Loss: 0.000607 Epoch 2/10 - Train Loss: 0.000402, Val Loss: 0.000609 Epoch 3/10 - Train Loss: 0.000389, Val Loss: 0.000615 Epoch 4/10 - Train Loss: 0.000384, Val Loss: 0.000604 Epoch 5/10 - Train Loss: 0.000382, Val Loss: 0.000608 Epoch 6/10 - Train Loss: 0.000381, Val Loss: 0.000606 Epoch 7/10 - Train Loss: 0.000382, Val Loss: 0.000605 Epoch 8/10 - Train Loss: 0.000381, Val Loss: 0.000610 Epoch 9/10 - Train Loss: 0.000381, Val Loss: 0.000607 Epoch 10/10 - Train Loss: 0.000381, Val Loss: 0.000605
In [ ]:
# loss curves
plt.figure(figsize=(10, 5))
plt.plot(range(1, num_epochs+1), train_losses, 'o-', label='Train')
plt.plot(range(1, num_epochs+1), val_losses, 's-', label='Validation')
plt.xlabel('Epoch')
plt.ylabel('MSE Loss')
plt.title('Training and Validation Loss')
plt.legend()
plt.grid(True)
plt.show()
3.3¶
In [ ]:
def train_model(config, train_loader, val_loader, device):
"""Train a ReturnPredictor with given config, return losses and best model state."""
model = ReturnPredictor(hidden_size=config['hidden_size'], dropout=config['dropout']).to(device)
criterion = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=config['lr'])
t_losses, v_losses = [], []
best_val = float('inf')
best_state = None
for epoch in range(config['epochs']):
model.train()
ep_loss = 0
for X, y in train_loader:
X, y = X.to(device), y.to(device)
optimizer.zero_grad()
loss = criterion(model(X), y)
loss.backward()
optimizer.step()
ep_loss += loss.item()
model.eval()
vl = 0
with torch.no_grad():
for X, y in val_loader:
X, y = X.to(device), y.to(device)
vl += criterion(model(X), y).item()
t_losses.append(ep_loss / len(train_loader))
v_losses.append(vl / len(val_loader))
if v_losses[-1] < best_val:
best_val = v_losses[-1]
best_state = {k: v.clone() for k, v in model.state_dict().items()}
model.load_state_dict(best_state)
return model, t_losses, v_losses, best_val
In [ ]:
# hyperparameter configurations to try
configs = [
{'hidden_size': 256, 'lr': 0.001, 'dropout': 0.2, 'epochs': 10, 'name': 'Base (256, lr=1e-3)'},
{'hidden_size': 512, 'lr': 0.0005, 'dropout': 0.3, 'epochs': 15, 'name': 'Wide (512, lr=5e-4)'},
{'hidden_size': 128, 'lr': 0.002, 'dropout': 0.1, 'epochs': 10, 'name': 'Narrow (128, lr=2e-3)'},
{'hidden_size': 256, 'lr': 0.0001, 'dropout': 0.2, 'epochs': 20, 'name': 'Slow (256, lr=1e-4)'},
]
results = []
all_models = {}
for i, cfg in enumerate(configs):
print(f"\n--- Config {i+1}: {cfg['name']} ---")
torch.manual_seed(42)
mdl, t_loss, v_loss, best_v = train_model(cfg, train_loader, val_loader, device)
all_models[cfg['name']] = {'model': mdl, 'train_losses': t_loss, 'val_losses': v_loss}
results.append({'Config': cfg['name'], 'Hidden': cfg['hidden_size'],
'LR': cfg['lr'], 'Dropout': cfg['dropout'],
'Epochs': cfg['epochs'], 'Best Val Loss': best_v})
print(f" Best val loss: {best_v:.6f}")
results_df = pd.DataFrame(results)
print("\n")
print(results_df.to_string(index=False))
--- Config 1: Base (256, lr=1e-3) ---
Best val loss: 0.000604
--- Config 2: Wide (512, lr=5e-4) ---
Best val loss: 0.000604
--- Config 3: Narrow (128, lr=2e-3) ---
Best val loss: 0.000604
--- Config 4: Slow (256, lr=1e-4) ---
Best val loss: 0.000606
Config Hidden LR Dropout Epochs Best Val Loss
Base (256, lr=1e-3) 256 0.0010 0.2 10 0.000604
Wide (512, lr=5e-4) 512 0.0005 0.3 15 0.000604
Narrow (128, lr=2e-3) 128 0.0020 0.1 10 0.000604
Slow (256, lr=1e-4) 256 0.0001 0.2 20 0.000606
In [ ]:
# compare loss curves across configurations
fig, axes = plt.subplots(2, 2, figsize=(14, 10))
axes = axes.flatten()
for ax, (name, data) in zip(axes, all_models.items()):
ax.plot(data['train_losses'], 'o-', label='Train', markersize=3)
ax.plot(data['val_losses'], 's-', label='Val', markersize=3)
ax.set_title(name)
ax.set_xlabel('Epoch')
ax.set_ylabel('MSE Loss')
ax.legend()
ax.grid(True)
plt.suptitle('Loss Curves for Different Hyperparameter Configs')
plt.tight_layout()
plt.show()
In [ ]:
# select the best model based on validation loss
best_config_name = results_df.loc[results_df['Best Val Loss'].idxmin(), 'Config']
best_model = all_models[best_config_name]['model']
print(f"Best config: {best_config_name}")
Best config: Narrow (128, lr=2e-3)
4¶
4.1¶
In [ ]:
def calculate_r2(y_true, y_pred, benchmark=None):
ss_res = np.sum((y_true - y_pred) ** 2)
if benchmark is not None:
ss_tot = np.sum((y_true - benchmark) ** 2)
else:
ss_tot = np.sum((y_true - np.mean(y_true)) ** 2)
return 1 - ss_res / ss_tot if ss_tot != 0 else 0.0
In [ ]:
# evaluate the best fine-tuned model on test set
best_model.eval()
test_preds, test_actuals = [], []
with torch.no_grad():
for X, y in test_loader:
X = X.to(device)
out = best_model(X)
test_preds.extend(out.cpu().numpy().flatten())
test_actuals.extend(y.numpy().flatten())
test_preds = np.array(test_preds)
test_actuals = np.array(test_actuals)
r2_is = calculate_r2(test_actuals, test_preds)
r2_oos = calculate_r2(test_actuals, test_preds, benchmark=0)
mse = np.mean((test_actuals - test_preds) ** 2)
print(f"Test MSE: {mse:.8f}")
print(f"Test R² (in-sample): {r2_is:.6f}")
print(f"Test R² (OOS, b=0): {r2_oos:.6f}")
Test MSE: 0.00136546 Test R² (in-sample): -0.000302 Test R² (OOS, b=0): 0.000534
In [ ]:
# actual vs predicted scatter
plt.figure(figsize=(8, 6))
plt.scatter(test_actuals, test_preds, alpha=0.3, s=10)
lims = [min(test_actuals.min(), test_preds.min()), max(test_actuals.max(), test_preds.max())]
plt.plot(lims, lims, 'r--', label='Perfect prediction')
plt.xlabel('Actual Return')
plt.ylabel('Predicted Return')
plt.title('Actual vs Predicted Daily Returns (Fine-tuned Model)')
plt.legend()
plt.grid(True)
plt.tight_layout()
plt.show()
4.2¶
In [ ]:
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, classification_report
# Fine-tuned model: binary prediction
ft_pred_binary = (test_preds > 0).astype(int)
ft_true_binary = (test_actuals > 0).astype(int)
ft_acc = accuracy_score(ft_true_binary, ft_pred_binary)
ft_f1 = f1_score(ft_true_binary, ft_pred_binary)
ft_prec = precision_score(ft_true_binary, ft_pred_binary)
ft_rec = recall_score(ft_true_binary, ft_pred_binary)
print("=== Fine-tuned Model (Binary) ===")
print(classification_report(ft_true_binary, ft_pred_binary, target_names=['Decrease', 'Increase']))
=== Fine-tuned Model (Binary) ===
precision recall f1-score support
Decrease 0.82 0.00 0.01 2455
Increase 0.50 1.00 0.67 2437
accuracy 0.50 4892
macro avg 0.66 0.50 0.34 4892
weighted avg 0.66 0.50 0.34 4892
In [ ]:
# --- FinBERT: binary prediction on test period ---
# match FinBERT sentiment to next-trading-day actual returns
finbert_test = news_w_sent[news_w_sent['publication_datetime'] >= '2020-01-01'].copy()
fb_true = []
fb_pred = []
for _, row in finbert_test.iterrows():
ticker = row['tickers']
date = row['publication_datetime']
future = price_df[(price_df['Date'] > date) & (price_df['ticker'] == ticker)]['Date']
if len(future) > 0:
next_date = future.iloc[0]
ret = price_df.loc[(price_df['Date'] == next_date) & (price_df['ticker'] == ticker), 'daily_return']
if len(ret) > 0:
fb_true.append(1 if ret.values[0] > 0 else 0)
fb_pred.append(1 if row['sentiment'] > 0 else 0)
fb_true = np.array(fb_true)
fb_pred = np.array(fb_pred)
fb_acc = accuracy_score(fb_true, fb_pred)
fb_f1 = f1_score(fb_true, fb_pred)
fb_prec = precision_score(fb_true, fb_pred)
fb_rec = recall_score(fb_true, fb_pred)
print("=== FinBERT Pretrained (Binary) ===")
print(classification_report(fb_true, fb_pred, target_names=['Decrease', 'Increase']))
=== FinBERT Pretrained (Binary) ===
precision recall f1-score support
Decrease 0.50 0.50 0.50 2455
Increase 0.49 0.49 0.49 2437
accuracy 0.49 4892
macro avg 0.49 0.49 0.49 4892
weighted avg 0.49 0.49 0.49 4892
In [ ]:
# comparison table
comp_df = pd.DataFrame({
'Model': ['FinBERT (Pretrained)', 'Fine-tuned NN'],
'Accuracy': [fb_acc, ft_acc],
'F1 Score': [fb_f1, ft_f1],
'Precision': [fb_prec, ft_prec],
'Recall': [fb_rec, ft_rec]
})
print(comp_df.to_string(index=False))
Model Accuracy F1 Score Precision Recall
FinBERT (Pretrained) 0.493459 0.490752 0.491560 0.489947
Fine-tuned NN 0.499591 0.665482 0.498873 0.999179
5¶
5.1¶
In [ ]:
def calculate_sp_return(month):
"""Return of S&P500 in month t+1, based on last close of month t and month t+1."""
current = price_df[(price_df['Date'] >= month.start_time) &
(price_df['Date'] < (month + 1).start_time) &
(price_df['ticker'] == 'SPX')]
nxt = price_df[(price_df['Date'] >= (month + 1).start_time) &
(price_df['Date'] < (month + 2).start_time) &
(price_df['ticker'] == 'SPX')]
if not current.empty and not nxt.empty:
return (nxt['close'].iloc[-1] / current['close'].iloc[-1]) - 1
return None
def sentiment_market_analysis(news_subset, sentiment_col, model_name):
"""Monthly aggregation + scatter with OLS + time series plot."""
import statsmodels.api as sm
df = news_subset.copy()
df['publication_datetime'] = pd.to_datetime(df['publication_datetime'])
monthly = df.groupby(df['publication_datetime'].dt.to_period('M'))[sentiment_col].mean().reset_index()
monthly.columns = ['month', 'sentiment']
monthly['SP_return'] = monthly['month'].apply(calculate_sp_return)
monthly = monthly.dropna()
# scatter + OLS regression
X = sm.add_constant(monthly['sentiment'])
ols = sm.OLS(monthly['SP_return'], X).fit()
fig, axes = plt.subplots(1, 2, figsize=(16, 5))
# scatter plot
ax = axes[0]
ax.scatter(monthly['sentiment'], monthly['SP_return'], alpha=0.6)
x_range = np.linspace(monthly['sentiment'].min(), monthly['sentiment'].max(), 100)
ax.plot(x_range, ols.params[0] + ols.params[1] * x_range, 'r-', label=f'OLS (R²={ols.rsquared:.3f}, p={ols.pvalues[1]:.3f})')
ax.axhline(0, color='gray', linestyle='--', alpha=0.5)
ax.set_xlabel(f'Month t Sentiment ({model_name})')
ax.set_ylabel('S&P 500 Return (Month t+1)')
ax.set_title(f'{model_name}: Sentiment vs Next-Month Market Return')
ax.legend()
ax.grid(True)
# time series
ax2 = axes[1]
ax2.plot(monthly['month'].astype(str), monthly['sentiment'], 'o-', color='blue', label='Sentiment')
ax2b = ax2.twinx()
ax2b.plot(monthly['month'].astype(str), monthly['SP_return'], 'x-', color='orange', label='S&P 500 Return')
ax2.set_xlabel('Month')
ax2.set_ylabel('Sentiment', color='blue')
ax2b.set_ylabel('S&P 500 Return', color='orange')
ax2.set_title(f'{model_name}: Sentiment and Market Return Over Time')
ax2.tick_params(axis='x', rotation=45)
lines1, labels1 = ax2.get_legend_handles_labels()
lines2, labels2 = ax2b.get_legend_handles_labels()
ax2.legend(lines1 + lines2, labels1 + labels2, loc='upper left')
ax2.grid(True)
plt.tight_layout()
plt.show()
print(f"\n{model_name} - OLS Regression Summary:")
print(f" Slope: {ols.params[1]:.6f} (p={ols.pvalues[1]:.4f})")
print(f" R²: {ols.rsquared:.4f}")
print(f" Corr: {monthly['sentiment'].corr(monthly['SP_return']):.4f}")
return monthly, ols
In [ ]:
# FinBERT: month t sentiment vs month t+1 S&P500 return
finbert_test_data = news_w_sent[news_w_sent['publication_datetime'] >= '2020-01-01'].copy()
fb_monthly, fb_ols = sentiment_market_analysis(finbert_test_data, 'sentiment', 'FinBERT')
FinBERT - OLS Regression Summary: Slope: -0.307701 (p=0.3138) R²: 0.1011 Corr: -0.3180
5.2¶
In [ ]:
# fine-tuned model: month t sentiment vs month t+1 return
ft_test_data = news_df[test_mask].copy()
ft_test_data['ft_sentiment'] = test_preds
ft_monthly, ft_ols = sentiment_market_analysis(ft_test_data, 'ft_sentiment', 'Fine-tuned NN')
Fine-tuned NN - OLS Regression Summary: Slope: 42.764623 (p=0.9472) R²: 0.0005 Corr: 0.0214
5.3¶
In [ ]:
# side-by-side comparison
import statsmodels.api as sm
trend_comp = pd.DataFrame({
'Model': ['FinBERT', 'Fine-tuned NN'],
'OLS Slope': [fb_ols.params[1], ft_ols.params[1]],
'Slope p-value': [fb_ols.pvalues[1], ft_ols.pvalues[1]],
'OLS R²': [fb_ols.rsquared, ft_ols.rsquared],
'Correlation': [
fb_monthly['sentiment'].corr(fb_monthly['SP_return']),
ft_monthly['sentiment'].corr(ft_monthly['SP_return'])
]
})
print("Sentiment-Market Trend Comparison:")
print(trend_comp.to_string(index=False))
Sentiment-Market Trend Comparison:
Model OLS Slope Slope p-value OLS R² Correlation
FinBERT -0.307701 0.313787 0.101125 -0.318001
Fine-tuned NN 42.764623 0.947247 0.000460 0.021449
In [ ]:
# overlay both models' monthly sentiment
fig, ax = plt.subplots(figsize=(12, 5))
ax.plot(fb_monthly['month'].astype(str), fb_monthly['sentiment'], 'o-', label='FinBERT')
ax.plot(ft_monthly['month'].astype(str), ft_monthly['sentiment'], 's-', label='Fine-tuned NN')
ax.set_xlabel('Month')
ax.set_ylabel('Average Sentiment')
ax.set_title('Monthly Sentiment Comparison: FinBERT vs Fine-tuned')
ax.legend()
ax.grid(True)
plt.xticks(rotation=45)
plt.tight_layout()
plt.show()
