Merge pull request #38 from koheiw/dev-doc2vec

Update README

Author: koheiw

README.Rmd

@@ -14,7 +14,7 @@ knitr::opts_chunk$set(
 
 # Wordvector: word and document vector models
 
-The **wordvector** package is developed to create word and document vectors using **quanteda**. This package currently supports word2vec ([Mikolov et al., 2013](http://arxiv.org/abs/1310.4546)) and latent semantic analysis ([Deerwester et al., 1990](https://doi.org/10.1002/(SICI)1097-4571(199009)41:6<391::AID-ASI1>3.0.CO;2-9)). 
+The **wordvector** package is developed to create word and document vectors using **quanteda**. This package currently supports word2vec ([Mikolov et al., 2013](http://arxiv.org/abs/1310.4546)), doc2vec ([Le, Q. V., & Mikolov, T., 2014](https://doi.org/10.48550/arXiv.1405.4053)) and latent semantic analysis ([Deerwester et al., 1990](https://doi.org/10.1002/(SICI)1097-4571(199009)41:6<391::AID-ASI1>3.0.CO;2-9)). 
 
 ## How to install
 
@@ -69,7 +69,8 @@ wdv <- textmodel_word2vec(toks, dim = 50, type = "cbow", min_count = 5, verbose
 `similarity()` computes cosine similarity between word vectors.
 
 ```{r}
-head(similarity(wdv, c("amazon", "forests", "obama", "america", "afghanistan"), mode = "word"))
+head(similarity(wdv, c("amazon", "forests", "obama", "america", "afghanistan"), 
+                mode = "character"))
 ```
 
 ### Arithmetic operations of word vectors

README.md

@@ -3,8 +3,9 @@
 
 The **wordvector** package is developed to create word and document
 vectors using **quanteda**. This package currently supports word2vec
-([Mikolov et al., 2013](http://arxiv.org/abs/1310.4546)) and latent
-semantic analysis ([Deerwester et al.,
+([Mikolov et al., 2013](http://arxiv.org/abs/1310.4546)), doc2vec ([Le,
+Q. V., & Mikolov, T., 2014](https://doi.org/10.48550/arXiv.1405.4053))
+and latent semantic analysis ([Deerwester et al.,
 1990](https://doi.org/10.1002/(SICI)1097-4571(199009)41:6%3C391::AID-ASI1%3E3.0.CO;2-9)).
 
 ## How to install
@@ -41,7 +42,7 @@ download.file('https://www.dropbox.com/s/e19kslwhuu9yc2z/yahoo-news.RDS?dl=1',
 ``` r
 library(wordvector)
 library(quanteda)
-## Package version: 4.2.1
+## Package version: 4.3.1
 ## Unicode version: 15.1
 ## ICU version: 74.1
 ## Parallel computing: 16 of 16 threads used.
@@ -64,16 +65,16 @@ wdv <- textmodel_word2vec(toks, dim = 50, type = "cbow", min_count = 5, verbose
 ##  ...using 16 threads for distributed computing
 ##  ...initializing
 ##  ...negative sampling in 10 iterations
-##  ......iteration 1 elapsed time: 4.99 seconds (alpha: 0.0465)
-##  ......iteration 2 elapsed time: 10.00 seconds (alpha: 0.0431)
-##  ......iteration 3 elapsed time: 14.97 seconds (alpha: 0.0396)
-##  ......iteration 4 elapsed time: 19.94 seconds (alpha: 0.0362)
-##  ......iteration 5 elapsed time: 25.11 seconds (alpha: 0.0326)
-##  ......iteration 6 elapsed time: 30.14 seconds (alpha: 0.0291)
-##  ......iteration 7 elapsed time: 35.12 seconds (alpha: 0.0257)
-##  ......iteration 8 elapsed time: 40.22 seconds (alpha: 0.0222)
-##  ......iteration 9 elapsed time: 45.15 seconds (alpha: 0.0188)
-##  ......iteration 10 elapsed time: 50.32 seconds (alpha: 0.0152)
+##  ......iteration 1 elapsed time: 6.44 seconds (alpha: 0.0455)
+##  ......iteration 2 elapsed time: 13.22 seconds (alpha: 0.0408)
+##  ......iteration 3 elapsed time: 19.80 seconds (alpha: 0.0363)
+##  ......iteration 4 elapsed time: 26.97 seconds (alpha: 0.0317)
+##  ......iteration 5 elapsed time: 34.22 seconds (alpha: 0.0270)
+##  ......iteration 6 elapsed time: 41.09 seconds (alpha: 0.0224)
+##  ......iteration 7 elapsed time: 47.71 seconds (alpha: 0.0178)
+##  ......iteration 8 elapsed time: 54.47 seconds (alpha: 0.0131)
+##  ......iteration 9 elapsed time: 61.07 seconds (alpha: 0.0085)
+##  ......iteration 10 elapsed time: 67.54 seconds (alpha: 0.0041)
 ##  ...complete
 ```
 
@@ -82,21 +83,22 @@ wdv <- textmodel_word2vec(toks, dim = 50, type = "cbow", min_count = 5, verbose
 `similarity()` computes cosine similarity between word vectors.
 
 ``` r
-head(similarity(wdv, c("amazon", "forests", "obama", "america", "afghanistan"), mode = "word"))
-##      amazon        forests       obama            america          
-## [1,] "amazon"      "forests"     "obama"          "america"        
-## [2,] "rainforest"  "herds"       "barack"         "africa"         
-## [3,] "plantations" "rainforests" "biden"          "dakota"         
-## [4,] "farms"       "rainforest"  "kerry"          "american"       
-## [5,] "patagonia"   "plantations" "administration" "carolina"       
-## [6,] "warm-water"  "farmland"    "hagel"          "america-focused"
+head(similarity(wdv, c("amazon", "forests", "obama", "america", "afghanistan"), 
+                mode = "character"))
+##      amazon       forests       obama                   america          
+## [1,] "amazon"     "forests"     "obama"                 "america"        
+## [2,] "rainforest" "herds"       "biden"                 "america-focused"
+## [3,] "peat"       "rainforests" "relationship-building" "carolina"       
+## [4,] "re-grown"   "farmland"    "kerry"                 "american"       
+## [5,] "peatlands"  "rainforest"  "hagel"                 "dakota"         
+## [6,] "sunflower"  "forest"      "clinton"               "africa"         
 ##      afghanistan  
 ## [1,] "afghanistan"
 ## [2,] "afghan"     
-## [3,] "kabul"      
-## [4,] "pakistan"   
-## [5,] "taliban"    
-## [6,] "afghans"
+## [3,] "taliban"    
+## [4,] "kabul"      
+## [5,] "afghans"    
+## [6,] "pakistan"
 ```
 
 ### Arithmetic operations of word vectors
@@ -106,25 +108,25 @@ head(similarity(wdv, c("amazon", "forests", "obama", "america", "afghanistan"),
 ``` r
 # What is Amazon without forests?
 head(similarity(wdv, analogy(~ amazon - forests))) 
-##      [,1]         
-## [1,] "choo"       
-## [2,] "smash-hit"  
-## [3,] "yahoo"      
-## [4,] "tripadvisor"
-## [5,] "univision"  
-## [6,] "dreamworks"
+##      [,1]            
+## [1,] "yahoo"         
+## [2,] "smash-hit"     
+## [3,] "gawker"        
+## [4,] "aggregators"   
+## [5,] "troll"         
+## [6,] "globe-spanning"
 ```
 
 ``` r
 # What is for Afghanistan as Obama for America? 
 head(similarity(wdv, analogy(~ obama - america + afghanistan))) 
 ##      [,1]         
-## [1,] "taliban"    
-## [2,] "afghanistan"
-## [3,] "karzai"     
-## [4,] "hagel"      
-## [5,] "hamid"      
-## [6,] "obama"
+## [1,] "afghanistan"
+## [2,] "karzai"     
+## [3,] "afghan"     
+## [4,] "taliban"    
+## [5,] "obama"      
+## [6,] "nato"
 ```
 
 These examples replicates analogical tasks in the original word2vec
@@ -133,23 +135,23 @@ paper.
 ``` r
 # What is for France as Berlin for Germany?
 head(similarity(wdv, analogy(~ berlin - germany + france))) 
-##      [,1]       
-## [1,] "paris"    
-## [2,] "berlin"   
-## [3,] "bourget"  
-## [4,] "brussels" 
-## [5,] "amsterdam"
+##      [,1]        
+## [1,] "paris"     
+## [2,] "strasbourg"
+## [3,] "brussels"  
+## [4,] "berlin"    
+## [5,] "amsterdam" 
 ## [6,] "france"
 ```
 
 ``` r
 # What is for slowly as quick for quickly?
 head(similarity(wdv, analogy(~ quick - quickly + slowly)))
-##      [,1]             
-## [1,] "uneven"         
-## [2,] "gravity-defying"
-## [3,] "slow"           
-## [4,] "super-charged"  
-## [5,] "buck"           
-## [6,] "sideways"
+##      [,1]       
+## [1,] "uneven"   
+## [2,] "stumble"  
+## [3,] "backwards"
+## [4,] "fades"    
+## [5,] "slow"     
+## [6,] "upside"
 ```

tests/misc/compare_doc2vec.R

@@ -12,7 +12,7 @@ rownames(dat3) <- dat3$doc_id
 
 # doc2vec package -----------------------------
 
-d2v <- paragraph2vec(dat3, dim = 50, threads = 1, type = "PV-DM", trace = FALSE)
+d2v <- paragraph2vec(dat3, dim = 50, threads = 8, type = "PV-DBOW", trace = FALSE)
 mat_d2v <- as.matrix(d2v, which = "docs", normalize = FALSE)
 hist(mat_d2v["4362315",, drop = TRUE])
 
@@ -42,7 +42,7 @@ corp <- corpus(dat3)
 toks <- tokens(corp)
 dfmt <- dfm(toks, remove_padding = TRUE)
 options(wordvector_threads = 8)
-wdv <- textmodel_doc2vec(toks, dim = 50, type = "cbow", min_count = 5, verbose = TRUE, iter = 5,
+wdv <- textmodel_doc2vec(toks, dim = 50, type = "dbow2", min_count = 5, verbose = TRUE, iter = 5,
                          tolower = FALSE, alpha = 0.05)
 mat_wdv <- as.matrix(wdv, layer = "documents", normalize = FALSE)
 hist(mat_wdv["4362315",, drop = TRUE])
@@ -78,5 +78,7 @@ mean(diag(sim_wdv_all)) - mean(sim_wdv_all)
 plot(rowSums(sim_wdv), rowSums(sim_d2v))
 cor(rowSums(sim_wdv), rowSums(sim_d2v))
 
-
+hist(as.matrix(wdv, layer = "words", normalize = FALSE)[1,])
+hist(wdv$values$word[,1])
+hist(wdv$weights[,1])
 

tests/misc/probability.R

@@ -0,0 +1,37 @@
+library(quanteda)
+library(wordvector)
+options(wordvector_threads = 8)
+
+seed <- LSX::seedwords("sentiment")
+
+corp <- data_corpus_news2014
+toks <- tokens(corp, remove_punct = TRUE, remove_symbols = TRUE) %>%
+    tokens_remove(stopwords("en", "marimo"), padding = TRUE) %>%
+    tokens_select("^[a-zA-Z-]+$", valuetype = "regex", case_insensitive = FALSE,
+                  padding = TRUE) %>%
+    tokens_tolower()
+
+wov <- textmodel_word2vec(toks, dim = 100, type = "sg", min_count = 5, verbose = TRUE, iter = 10)
+dov <- textmodel_doc2vec(toks, dim = 100, type = "dbow", min_count = 5, verbose = TRUE, iter = 10)
+
+doc <- probability(dov, seed, layer = "documents")[,1]
+print(toks[head(doc)])
+print(toks[tail(doc)])
+
+cor(probability(wov, seed, layer = "words", mode = "numeric")[,1],
+    probability(dov, seed, layer = "words", mode = "numeric")[,1])
+
+lss0 <- LSX::as.textmodel_lss(wov, seed, spatial = FALSE)
+lss <- LSX:::as.textmodel_lss(dov, seed, spatial = FALSE)
+pred0 <- predict(lss0, newdata = dfm(toks, remove_padding = TRUE), rescale = FALSE,
+                 min_n = 40)
+pred <- predict(lss, newdata = dfm(toks, remove_padding = TRUE), rescale = FALSE,
+                 min_n = 40)
+pred2 <- probability(dov, seed, layer = "documents", mode = "numeric")[,1] / length(seed)
+
+dat <- data.frame(pred0, pred, pred2, pred3 = pred + pred2, text = corp[])
+
+plot(dat[,1:4])
+cor(dat[,1:4])
+View(dat)
+head(dat)