Predicting Stock Prices of Home Depot Based on Trends and the Sentiment of News and Tweets

import pandas as pd
import seaborn
import matplotlib.pyplot as plt
import datetime as dt
from datetime import date
from dateutil.relativedelta import relativedelta

!pip install yahoo_fin

from yahoo_fin import stock_info as si

def stock_data(stock_name):
    df = si.get_data(stock_name, start_date=(date.today() - relativedelta(years=3)).strftime('%m/%d/%Y'), 
                     end_date=date.today().strftime('%m/%d/%Y'))
    return df
ticker='HD'
company_name='Home Depot'
df = stock_data(ticker)
df.head()

df.tail()

#Add target values for two weeks and one month. 
#Dataset has values for weekdays only, so two weeks will be 10 data points
# and one month will be 20 data points
df['TwoWeeks'] = df['close'].rolling(10).mean()
df['Month'] = df['close'].rolling(20).mean()
df.head()

df.dropna(inplace=True)
df.head()

!pip install pytrends

import pytrends
from pytrends.request import TrendReq

pytrends = TrendReq()
kw_list=['Home Depot']

#Gather Google trends for ticker 'HD' and Home Depot
pytrends.build_payload(kw_list, geo='', timeframe='{} {}'.format((date.today() - relativedelta(years=3)).strftime('%Y-%m-%d'),date.today().strftime('%Y-%m-%d')))#timeframe='2010-06-29 2023-05-16')
keyword_interest = pytrends.interest_over_time()
del keyword_interest['isPartial']
keyword_interest.columns.rename("{}".format(ticker))
keyword_interest.head()

keyword_interest['Home Depot'].plot(title='Google Trends for \'Home Depot\' in the last three years')

df_combined = pd.concat([df, keyword_interest], axis=1)
#df_combined.dropna(subset='Month', inplace=True)
df_combined.head(20)

df_combined['Home Depot'] = df_combined['Home Depot'].fillna(method='ffill')
df_combined.dropna(inplace=True)
df_combined.head(30)

!pip3 install git+https://github.com/JustAnotherArchivist/snscrape.git

kw_list

# The following code collects data related to 'HD' and Home Depot from 
# "markets" and "MarketWatch" twitter accounts published in the last three years

import snscrape.modules.twitter as sntwitter
import pandas as pd
import datetime as dt
from datetime import date

# Creating list to append tweet data to
attributes_container = []

sources_all =['CNBC','cnn', 'cnnbrk','MarketWatch', 'Benzinga', 'Stocktwits','BreakoutStocks', 
          'bespokeinvest','WSJMarkets','Stephanie_Link','nytimesbusiness','IBDinvestors','WSJDealJournal',
         'business', 'TheEconomist','WSJ', 'ABC', 'CBSNews','FoxNews', 'NBCNews']

sources=['markets','MarketWatch']
for tweet in sntwitter.TwitterSearchScraper('from:{} since:{} until:{}'.format(sources[0], (date.today() - relativedelta(years=3)).strftime('%Y-%m-%d'), date.today().strftime('%Y-%m-%d'))).get_items():
    if 'Home Depot'.lower() in tweet.content.lower():
        attributes_container.append([tweet.date, tweet.content.split('http')[0]])
tweets_df = pd.DataFrame(attributes_container, columns=["Date Created", "Tweets"])
tweets_df.drop_duplicates(subset='Tweets',inplace=True)

attributes_container = []
for tweet in sntwitter.TwitterSearchScraper('from:{} since:{} until:{}'.format(sources[1], (date.today() - relativedelta(years=3)).strftime('%Y-%m-%d'), date.today().strftime('%Y-%m-%d'))).get_items():
    if 'Home Depot'.lower() in tweet.content.lower():
        attributes_container.append([tweet.date, tweet.content.split('http')[0]])
# Creating a dataframe from the tweets list above 
tweets_df_marketwatch = pd.DataFrame(attributes_container, columns=["Date Created", "Tweets"])
print(tweets_df_marketwatch.shape)
tweets_df_marketwatch.drop_duplicates(subset='Tweets',inplace=True)

pd.set_option("max_colwidth", None)
tweets_df.head()

tweets_df_marketwatch.shape

!pip install pygooglenews 

kw_list

import pygooglenews
from pygooglenews import GoogleNews
gn = GoogleNews()
headlines_related = []

date_list = pd.date_range(end=date.today().strftime('%Y-%m-%d'), start=(date.today()-relativedelta(years=3)).strftime('%Y-%m-%d')).tolist()
clean_date = [str(i).split(" ")[0] for i in date_list]
for date in clean_date:
    headlines = []
    for word in kw_list:
        search =gn.search(word,when=date)
        for item in search['entries']:
            headlines.append(item['title'])
    #We have headlines of the news whose body may contain one of the keywords.
    #We will use only headlines to make predictions but some headlines do not contain any of the keywords.
    #Only include headlines that has one of the keywords
    for headline in headlines:
        for i in range(len(kw_list)):
            if kw_list[i] in headline:
                if headline not in headlines_related:
                    headlines_related.append((date,headline))
len(headlines_related)

headlines_related[:10]

df_news = pd.DataFrame(headlines_related, columns=['Date', 'Headline'])
df_news['Headline'] = df_news['Headline'].apply(lambda x: x.split('-')[0])
df_news['Date'] = pd.to_datetime(df_news['Date'])
print(df_news.shape)
df_news.drop_duplicates(subset='Headline', inplace=True)
print(df_news.shape)
df_news.head()

!pip install vaderSentiment

tweets_df_marketwatch.drop("User Name", axis=1, inplace=True)

#combine tweets coming from markets, and marketwatch
markets = pd.concat([tweets_df, tweets_df_marketwatch])
markets.shape

from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
sid = SentimentIntensityAnalyzer() 
def vader(row):
    comp = sid.polarity_scores(row)['compound']
    return comp

import re
markets['Tweets'] = markets['Tweets'].apply(lambda x: re.sub("amp;","&", x))
markets['Tweets'] = markets['Tweets'].apply(lambda x: re.sub("\n"," ", x))
markets['Tweets'] = markets['Tweets'].apply(lambda x: re.sub("\xa0"," ", x))
markets.head()

markets['Date Created'] = markets['Date Created'].astype('str')

markets['Date Created'] = markets['Date Created'].apply(lambda x:x[:11])
markets.head()

markets['Date Created'] = pd.to_datetime(markets['Date Created'])
markets.sort_values(by='Date Created', inplace=True)
markets.index =range(markets.shape[0])
markets.head()

markets['VaderSent'] = markets['Tweets'].apply(vader)
markets

#Vader gives a sentiment value between -1 and 1, -1 being the most negative, 
#1 being the most positive and 0 being neutral. We will put them in the corresponding
# pos, neu or neg bins based on this value.
def bins(value):
    hold_sent = []
    if -1 <= value < -0.33:
        hold_sent.append('neg')
    if -0.33<=value<=0.33:
        hold_sent.append('neu')
    if 0.33 < value <= 1:
        hold_sent.append('pos')
    return hold_sent
markets['sentiment'] = markets['VaderSent'].apply(bins)
values=[]
for i in markets['sentiment'].values:
    values.append(i[0])
markets['sentiment']=values
markets.head(10)

#Each week determine the weight of positive, neutral and negative tweets:
tweets_weight = pd.DataFrame(markets.groupby([pd.Grouper(key='Date Created', freq='W')])['sentiment'], columns=['Date','Data'])
tweets_weight.head()
for i in range(len(tweets_weight)):
    n = dict(tweets_weight.loc[i,'Data'].value_counts(normalize=True))
    if 'neg' not in n:
        n['neg']=0
    if 'pos' not in n:
        n['pos']=0
    if 'neu' not in n:
        n['neu']=0
    tweets_weight.loc[i,'tweet_pos'] = n['pos']
    tweets_weight.loc[i,'tweet_neg'] = n['neg']
    tweets_weight.loc[i,'tweet_neu'] = n['neu']
tweets_weight

tweets_weight.drop('Data', inplace=True, axis=1)
tweets_weight.head()

tweets_weight.plot(x='Date', y=['tweet_pos','tweet_neu','tweet_neg'],title='Tweets\' Sentiment')

df_news['VaderSent'] = df_news['Headline'].apply(vader)
df_news.head(10)

df_news['Date'] = pd.to_datetime(df_news['Date'])

df_news['sentiment'] = df_news['VaderSent'].apply(bins)
values=[]
for i in df_news['sentiment'].values:
    values.append(i[0])
df_news['sentiment']=values
df_news.head(10)

news_weight = pd.DataFrame(df_news.groupby([pd.Grouper(key='Date', freq='W')])['sentiment'], columns=['Date','Data'])
for i in range(len(news_weight)):
    n = dict(news_weight.loc[i,'Data'].value_counts(normalize=True))
    if 'neg' not in n:
        n['neg']=0
    if 'pos' not in n:
        n['pos']=0
    if 'neu' not in n:
        n['neu']=0
    news_weight.loc[i,'news_pos'] = n['pos']
    news_weight.loc[i,'news_neg'] = n['neg']
    news_weight.loc[i,'news_neu'] = n['neu']
news_weight

news_weight.drop('Data', inplace=True, axis=1)
news_weight

news_weight.plot(x='Date', y=['news_pos','news_neg','news_neu'], title='News Sentiment')

df_combined.drop('ticker', inplace=True, axis=1)
df_combined.head()

df_combined = df_combined.reset_index()

df_combined.rename(columns={'index':'Date'}, inplace=True)
df_combined.head()

import numpy as np
df_weekly = pd.DataFrame(df_combined.groupby([pd.Grouper(key='Date', freq='W')]).agg(np.mean))
df_weekly.head()

df_final = df_weekly.merge(news_weight, on='Date').merge(tweets_weight, on='Date')
df_final.columns = [i.lower() for i in df_final.columns]
df_final.head()

df_final.rename(columns={"home depot":"trends"}, inplace=True)
df_final.head()

#Scaling the data
from sklearn.preprocessing import StandardScaler
SS = StandardScaler()
df_final_scaled = pd.DataFrame(SS.fit_transform(df_final.iloc[:,1:]), columns=df_final.columns[1:])
df_final_scaled.insert(0, "date", df_final['date'])
df_final_scaled.head()

df_final_scaled.corr().round(3)

cols=['news_pos', 'news_neg','news_neu', 'tweet_pos', 'tweet_neg','tweet_neu', 'trends']
fig, ax =plt.subplots(3,3, figsize=(15, 12))
ax=ax.ravel()
for i in range(len(cols)):
    df_final_scaled.plot(x='date', y=['close','twoweeks','month', cols[i]], ax=ax[i], title=cols[i])
    ax[7].axis('off')
    ax[8].axis('off')
plt.tight_layout()

%matplotlib inline
import matplotlib.pyplot as plt
import pandas as pd
plt.style.use("ggplot")

df_final_scaled = pd.read_csv('df_final_home_depot_scaled.csv')
df_final_scaled.drop('Unnamed: 0', inplace=True, axis=1)
df_final_scaled.head()

x= df_final_scaled[['open','high','low','close', 'adjclose','volume','news_pos', 'news_neg', 'news_neu', 'tweet_pos',
       'tweet_neg', 'tweet_neu', 'trends']]
y=df_final_scaled['twoweeks']

import statsmodels.api as sm
x = sm.add_constant(x)
model = sm.OLS(y, x).fit()
print(model.summary())

x= df_final_scaled[['open','high','low','close', 'adjclose','volume']]
y=df_final_scaled['twoweeks']

from sklearn.model_selection import train_test_split 
train_x, test_x, train_y, test_y = train_test_split(x, y, test_size=0.3 , shuffle=False,random_state = 0)

from sklearn.linear_model import LinearRegression
from sklearn.metrics import confusion_matrix, accuracy_score
regression = LinearRegression()
regression.fit(train_x, train_y)
print("regression coefficient",regression.coef_)
print("regression intercept",regression.intercept_)

regression_confidence = regression.score(test_x, test_y)
print("linear regression confidence: ", regression_confidence)

predicted=regression.predict(test_x)
print(test_x.head())

dfr=pd.DataFrame({'Actual_Price':test_y, 'Predicted_Price':predicted})
dfr.head(10)

dfr.describe()

from sklearn import metrics
import numpy as np
print('Mean Absolute Error (MAE):', metrics.mean_absolute_error(test_y, predicted))
print('Mean Squared Error (MSE) :', metrics.mean_squared_error(test_y, predicted))
print('Root Mean Squared Error (RMSE):', np.sqrt(metrics.mean_squared_error(test_y, predicted)))

plt.scatter(dfr.Actual_Price, dfr.Predicted_Price,  color='Darkblue')
plt.xlabel("Actual Price")
plt.ylabel("Predicted Price")
plt.show()

plt.plot(dfr.Actual_Price, color='black', label='Actual')
plt.plot(dfr.Predicted_Price, color='blue',label='Predicted')
plt.title("Prediction using only Historical Data")
plt.legend()

df_final_scaled.columns

x= df_final_scaled[['open','high','low','close', 'adjclose','volume','news_pos', 'news_neg', 'news_neu', 'tweet_pos',
       'tweet_neg', 'tweet_neu', 'trends']]

y=df_final_scaled['twoweeks']

train_x, test_x, train_y, test_y = train_test_split(x, y, test_size=0.3 , shuffle=False,random_state = 0)
regression = LinearRegression()
regression.fit(train_x, train_y)
print("regression coefficient",regression.coef_)
print("regression intercept",regression.intercept_)

regression_confidence = regression.score(test_x, test_y)
print("linear regression confidence: ", regression_confidence)

predicted=regression.predict(test_x)
print(test_x.head())

from sklearn.metrics import r2_score
r2_score(predicted, test_y)

dfr=pd.DataFrame({'Actual_Price':test_y, 'Predicted_Price':predicted})
dfr.head(10)

dfr.describe()

print('Mean Absolute Error (MAE):', metrics.mean_absolute_error(test_y, predicted))
print('Mean Squared Error (MSE) :', metrics.mean_squared_error(test_y, predicted))
print('Root Mean Squared Error (RMSE):', np.sqrt(metrics.mean_squared_error(test_y, predicted)))

plt.scatter(dfr.Actual_Price, dfr.Predicted_Price,  color='Darkblue')
plt.xlabel("Actual Price")
plt.ylabel("Predicted Price")
plt.show()

plt.plot(dfr.Actual_Price, color='black', label='Actual')
plt.plot(dfr.Predicted_Price, color='blue',label='Predicted')
plt.title("Predictions using  Historical, Sentiment and Trends")
plt.legend()

import pandas as pd
df = pd.read_csv('df_final_home_depot_scaled.csv')
df.drop('Unnamed: 0', inplace=True, axis=1)
df.head()

df.columns

df_hist = df[['date', 'open', 'high', 'low', 'close', 'adjclose', 'volume',
       'twoweeks', 'month']]
df_hist.shape

#Append each row to X (high, low, open, close, adjclose, volume)
X = [[df_hist.iloc[j,i+1] for i in range(df_hist.shape[1]-3)] for j in range(df_hist.shape[0])]
#Append all "twoweeks" values
Y = [df_hist.iloc[i,7] for i in range(df_hist.shape[0])]

import numpy as np
from sklearn.model_selection import train_test_split 


x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size=0.3, shuffle=True)

train_X = np.array(x_train)
test_X = np.array(x_test)
train_Y = np.array(y_train)
test_Y = np.array(y_test)
train_X = train_X.reshape(train_X.shape[0],1,6,1)
test_X = test_X.reshape(test_X.shape[0],1,6,1)
print(len(train_X))
print(len(test_X))

train_X[0]

from tensorflow.keras import backend as K
K.image_data_format()=="channels_first"

# For creating model and training
import tensorflow as tf
from tensorflow.keras.layers import Conv1D, LSTM, Dense, Dropout, Bidirectional, TimeDistributed
from tensorflow.keras.layers import MaxPooling1D, Flatten
from tensorflow.keras.regularizers import L1, L2
from tensorflow.keras.metrics import Accuracy
from tensorflow.keras.metrics import RootMeanSquaredError

model = tf.keras.Sequential()

# Creating the Neural Network model here...
# CNN layers
model.add(TimeDistributed(Conv1D(64, kernel_size=2, activation='relu', input_shape=(None, 6, 1))))
model.add(TimeDistributed(MaxPooling1D(1)))
model.add(TimeDistributed(Conv1D(128, kernel_size=2, activation='relu')))
model.add(TimeDistributed(MaxPooling1D(1)))
model.add(TimeDistributed(Conv1D(64, kernel_size=2, activation='relu')))
model.add(TimeDistributed(MaxPooling1D(1)))
model.add(TimeDistributed(Flatten()))
# model.add(Dense(5, kernel_regularizer=L2(0.01)))

# LSTM layers
model.add(Bidirectional(LSTM(100, return_sequences=True)))
model.add(Dropout(0.5))
model.add(Bidirectional(LSTM(100, return_sequences=False)))
model.add(Dropout(0.5))

#Final layers
model.add(Dense(1, activation='linear'))
model.compile(optimizer='adam', loss='mse', metrics=['mse', 'mae'])

history = model.fit(train_X, train_Y, validation_data=(test_X,test_Y), epochs=40,batch_size=40, verbose=1, shuffle =True)

import matplotlib.pyplot as plt
plt.plot(history.history['loss'], label='train loss')
plt.plot(history.history['val_loss'], label='val loss')
plt.xlabel("epoch")
plt.ylabel("Loss")
plt.legend()

plt.plot(history.history['mse'], label='train mse')
plt.plot(history.history['val_mse'], label='val mse')
plt.xlabel("epoch")
plt.ylabel("Loss")
plt.legend()