library(rvest)
library(tidyverse)
library(RSelenium)
library(syuzhet)
library(stargazer)

mx = function(myVector){
  which( myVector == max(myVector) )
}

countwords = function(str1){
  lengths(gregexpr("\\W+", str1)) + 1
}

url = 'https://www.gettr.com/trending'

rD = rsDriver(browser = "firefox",
              port = 4252L,
              verbose = F)
remDr = rD$client

## get the html source 
remDr$navigate(url)
Sys.sleep(1)
for(i in 1:5){
  webElem = remDr$findElement("css", "body")
  webElem$sendKeysToElement(list(key = 'end' ))#scroll down
  Sys.sleep(.75)
  # print(i)
}
src = remDr$getPageSource()[[1]]

classes = src %>%
  read_html %>%
  html_nodes("*") %>%
  html_attr("class") %>%
  unique %>%
  str_subset('timeline') %>%
  str_replace_all('\\s', '.')

## Fetch the posts and relevant info
posts = c()
for (i in 1:length(classes)) {
  post = src %>%
    read_html %>%
    html_nodes(paste0('div.', classes[i])) %>%
    html_text()
  posts = c(posts, post)
}

dates = posts %>%
  str_subset('\\n') %>%
  str_extract('·\\w+\\s\\d{1,2},\\s\\d{4}') %>%
  str_remove_all('·')

comments = posts %>%
  str_subset('\\n') %>%
  str_extract('\\d{1,100}Like') %>%
  str_remove_all('Like')

likes = posts %>%
  str_subset('\\n') %>%
  str_extract('Like.+Repost') %>%
  str_remove_all('Like|Repost')

likes[grepl('K', likes)] = (likes[grepl('K', likes)] %>% parse_number()) *
  1000
likes = likes %>% parse_number()

shares = posts %>%
  str_subset('\\n') %>%
  str_extract('Repost.+Share') %>%
  str_remove_all('Share|Repost')

nrc = c()

for (i in 1:((posts %>% str_subset('\\n')) %>% length)) {
  senti = ((posts %>%
              str_subset('\\n'))[i]) %>%
    str_remove_all(paste0('\\s', tm::stopwords(), '\\s', collapse = '|')) %>%
    str_extract('\\n.+') %>%
    str_remove_all('\\n') %>%
    str_split('\\s') %>%
    unlist %>%
    get_nrc_sentiment() %>%
    colSums()
  senti = c(senti, total = senti %>% sum)

  nrc = rbind(nrc, senti)
}

df = data.frame(dates, comments, likes, shares, nrc)
rownames(df) = NULL

## Fetch the most mentioned websites
websites = posts %>%
  str_extract('www\\S+com') %>%
  na.omit() %>%
  table %>%
  sort %>% as.data.frame()

## Fetch the news sidebar section
news.senti = src %>% 
  read_html() %>% 
  html_nodes('div.sidebar-body') %>% 
  html_text() %>% 
  str_extract_all('\\w+') %>% 
  unlist %>% 
  get_nrc_sentiment()

lbs = nrc[,1:10] %>% colnames()
posts.news = rbind.data.frame(
  cbind((nrc[,1:10] %>% colSums)/nrow(nrc), 'Posts',lbs ),
  cbind((news.senti %>% colSums)/nrow(nrc), 'News',lbs ))
posts.news[,1] = posts.news[,1] %>% as.numeric()
posts.news[,2] = posts.news[,2] %>% as.factor()
posts.news[,3] = posts.news[,3] %>% as.factor()

## Wrangle the data for a (probably faulty) lm
lm.data2 = c() 
for (i in 1:10){
  entry = cbind(likes,nrc[,i], colnames(nrc)[i])
  lm.data2 = rbind(lm.data2, entry)
}
lm.data2 = data.frame(lm.data2)
colnames(lm.data2) = c('Likes','Score','Emotion')
lm.data2[,1] = lm.data2[,1] %>% as.numeric()
lm.data2[,2] = lm.data2[,2] %>% as.numeric()
lm.data2 = lm.data2 %>% 
  filter(Emotion != 'positive') %>% 
  filter(Emotion!='negative')
lm.data2[,3] = lm.data2[,3] %>% as.factor()

lm2 = lm(Likes ~ .*., lm.data2)

##  Get the overall frequency of sentiments
overall.freq =  data.frame(colSums(nrc[, 1:10]), colSums(nrc[, 1:10]) %>% names)

## Wrangle the data for a (better) lm; this includes looking for the predominant sentiment(s) in each post
lm.data = data.frame(l = df$likes, nrc)
for (i in 2:ncol(lm.data)){
  lm.data[,i] = lm.data[,i] %>% scales::rescale()
}

factors = c()
for (i in 1:length(apply(lm.data[,2:11], 1, mx))){
  factors = c( factors, apply(lm.data[,2:11], 1, mx)[[i]] %>% names %>% str_extract_all('^\\w{1,3}|\\s\\w{1,3}') %>% paste(collapse = '-') )
}

lm.data = data.frame(likes = df$likes,
                     Sentiments = factors 
)[(sapply(factors, countwords)<5),]
lm.data[,2] = lm.data[,2] %>% as.factor()
lm = lm(likes ~ .*., lm.data)

## Look for the frq of the predominant sentiment(s)
complex.sent = data.frame(
  table(lm.data[,2]) %>% sort,table(lm.data[,2]) %>% sort %>% names 
)

rD$server$stop()

ggplot(websites) +
  aes(x = ., fill = ., weight = Freq) +
  geom_bar(alpha = .75, col = 1) +
  scale_fill_hue(direction = 1) +
  labs(x = "Links", y = "Freq") +
  theme_minimal() +
  theme(legend.position = "none")

websites %>% head %>% knitr::kable()

ggplot(posts.news) +
  aes(x = lbs, fill = V2, weight = V1) +
  geom_bar(alpha = .75, col = 1) +
  scale_fill_hue(direction = 1) +
  labs(x = "Sentiments", y = "Scores") +
  theme_minimal() 

posts.news %>% head %>% knitr::kable()

ggplot(lm.data2) +
  aes(x = Score, y = Likes, colour = Emotion) +
  geom_jitter(shape = "circle", size = 2, width = .2, alpha = .75 ) +
  scale_color_hue(direction = 1) +
  theme_minimal() 

lm2 = lm(Likes ~ .*., lm.data2)
ggeffects::ggpredict(lm2,
                     c('Score','Emotion'),
                     ci = F
) %>% plot

lm.data2 %>% head %>% knitr::kable()

stargazer(lm2, type = 'html')

ggplot(overall.freq) +
  aes(
    x = colSums.nrc...1.10.......names,
    fill = colSums.nrc...1.10.......names,
    weight = colSums.nrc...1.10..
  ) +
  geom_bar(alpha = .75, col = 1) +
  scale_fill_hue(direction = 1) +
  labs(x = "Sentiments", y = "Freq") +
  theme_minimal() +
  theme(legend.position = "none")

overall.freq %>% head %>% knitr::kable()

ggplot(complex.sent) +
  aes(x = Var1, fill = Var1, weight = Freq) +
  geom_bar(alpha = .75, col = 1) +
  scale_fill_hue(direction = 1) +
  labs(x = "Main sentiment(s)", y = "Freq") +
  theme_minimal() +
  theme(legend.position = "none")

complex.sent %>% head %>% knitr::kable()

ggplot(lm.data, mapping = aes(y = likes, Sentiments, col = Sentiments, fill =  Sentiments) )+
  geom_boxplot(alpha = .25)+
  ylab('Likes')+theme_bw()+
  theme(legend.position = "none")

ggeffects::ggpredict(lm,
                     c('Sentiments')) %>% plot(
                       jitter = 0
                     )

lm.data %>% head %>% knitr::kable()

stargazer(lm, type = 'html')