Neural networks, natural language processing, and earning call transcripts

library(quantmod)
library(knitr)
library(tm)
library(quanteda)
library(tidyverse)
library(rvest)
library(pins)

b = board_s3(
  '',
  region = 'us-west-1',
  access_key = '',
  secret_access_key = ''
)

## NRC lexicon
download.file('https://raw.githubusercontent.com/mjockers/syuzhet/master/R/sysdata.rda',destfile = 'syu.dic.rda')
load('syu.dic.rda')

countwords = function(str1){
  lengths(gregexpr("\\W+", str1)) + 1
}

## Gets text from a link
getText = function(link){
  try({
    link %>% 
      read_html() %>% 
      html_nodes('p') %>% 
      html_text() %>% 
      paste(collapse = '. ') %>% 
      str_replace_all('\\.\\.','\\.')
  })
}

## Gets date from an article given a link
getInfo = function(url){
  try({  attrs = url %>% 
    read_html() %>% 
    html_nodes('span') %>% 
    html_attrs()
  date = (url %>% 
            read_html() %>% 
            html_nodes('span') %>% 
            html_text())[match(1, (sapply(attrs, names) %>% str_extract_all('data') %>% sapply(length)))]
  t = url %>% 
    read_html() %>% 
    html_nodes('h1') %>% 
    html_text() 
  return(cbind(t,date))})
}

vec = paste0("https://seekingalpha.com/article/",4478500:4479741)
info = vec %>% lapply(getInfo)

##get dates
info2 = c()
web = c()
for(i in 1:length(info)){

  if(length(info[[i]]) == 2){
    web = c(web , i )
    info2 = rbind(info2 ,info[[i]])
  }

}

 ## get names of tickers
t = info2[,1] %>% 
  str_extract_all('[(][A-Z]{1,5}[)]') %>%
  unlist %>% 
  str_remove_all('\\(|\\)')

ind = info2[,1] %>% 
  str_which('[(][A-Z]{1,5}[)]') %>%
  unlist

df = cbind( info2[ind,1:2], t)

##transform into stem terms
txt = vec[web][ind] %>% sapply(getText) %>% sapply( stemDocument)
nwords = sapply( txt, countwords)
ind = which(nwords>500) %>% unname

mat = dfm(txt[ind])
mat = mat[,((mat %>% colnames() %>% tolower()) %in% syuzhet_dict[,1])] 

##get dates
adj =  (df[,2] %>% 
          str_extract('Jan.+2022') %>% 
          str_remove_all('Jan..|......$') %>% 
          parse_number())

##get change in prices before and after transcript was released
prices = c()
for ( i in 1:length(adj)){
  if(adj[i]!=14){
    entry = NA
    try({
      symb = getSymbols(df[i,3],auto.assign = F) %>% tail(n = 9)
      dates = symb %>% time %>% lubridate::day()
      entry =  ((symb[findInterval( adj[i], dates )+1,1][[1]])-
        (symb[findInterval( adj[i], dates ),1])[[1]])/ symb[findInterval( adj[i], dates ),1][[1]]

    })
  } else{
    entry = NA
  }
  prices = c(prices, entry)
}

##define input and output variables
predictor = mat[!is.na(prices[ind]),] %>% as.matrix()
target = prices[ind] %>% na.omit() %>% scales::rescale()

##scale the input variables
for( i in 1:ncol(predictor)){
  predictor[,i] = predictor[,i]%>% scales::rescale()
}

pin_write(b, predictor, 'nlpnn.data')

## EXAMPLE OF INPUT #####################################################
predictor %>% head %>% knitr::kable()