vignettes/pkgdown/examples/chinese.Rmd
chinese.Rmdlibrary(quanteda)Download corpus constructed from Report on the Work of the Government published by Premier of the State Council between 1954 and 2017. You can download the corpus using the quanteda.corpora package.
# read text files
devtools::install_github("quanteda/quanteda.corpora")library(quanteda.corpora)
corp <- quanteda.corpora::download(url = "https://www.dropbox.com/s/37ojd5knz1qeyul/data_corpus_chinesegovreport.rds?dl=1")# Chinese stopwords
ch_stop <- stopwords("zh", source = "misc")
# tokenize
ch_toks <- corp %>%
tokens(remove_punct = TRUE) %>%
tokens_remove(pattern = ch_stop)
# construct a dfm
ch_dfm <- dfm(ch_toks)
topfeatures(ch_dfm)## 发展 经济 社会 建设 改革 人民 主义 工作 企业 国家
## 5627 5036 4255 4248 2931 2897 2817 2642 2627 2595
# plot a word cloud
set.seed(100)
# to set the font correctly for macOS
library("quanteda.textplots")
textplot_wordcloud(ch_dfm, min_count = 500, random_order = FALSE,
rotation = .25, max_words = 100,
min_size = 0.5, max_size = 2.8,
font = if (Sys.info()['sysname'] == "Darwin") "SimHei" else NULL,
color = RColorBrewer::brewer.pal(8, "Dark2"))
# fcm within the window size of 5
ch17_corp <- corpus_subset(corp, Year == "2017")
ch17_toks <-
tokens(ch17_corp, remove_punct = TRUE) %>%
tokens_remove(ch_stop)
ch_fcm <- fcm(ch17_toks, context = "window")
topfeatures(ch_fcm["改革", ])## 推进 制度 性 体制 完善 试 点 供给 侧 结构
## 23 19 11 11 10 8 8 7 7 7
library("quanteda.textmodels")
wf <- textmodel_wordfish(ch_dfm)
y <- 1954:2017
y <- y[y <= 1964 | y >= 1975]
y <- y[!y %in% c(1963, 1961, 1962, 1976, 1977)]
plot(y, wf$theta, xlab = "Year", ylab = "Position")
# bigrams cross the whole dataset
library("quanteda.textstats")
ch_col <- textstat_collocations(ch_toks, size = 2, min_count = 20)
knitr::kable(head(ch_col, 10))| collocation | count | count_nested | length | lambda | z |
|---|---|---|---|---|---|
| 社会 主义 | 1787 | 0 | 2 | 5.667646 | 128.76439 |
| 亿 元 | 689 | 0 | 2 | 7.451647 | 93.09479 |
| 现代 化 | 632 | 0 | 2 | 6.956866 | 83.62882 |
| 体制 改革 | 504 | 0 | 2 | 5.199483 | 77.47483 |
| 五年 计划 | 341 | 0 | 2 | 5.365465 | 71.73289 |
| 各级 政府 | 306 | 0 | 2 | 6.116987 | 66.70430 |
| 增长 百分 | 300 | 0 | 2 | 5.527159 | 65.95692 |
| 万 吨 | 212 | 0 | 2 | 6.596337 | 62.62405 |
| 国民 经济 | 589 | 0 | 2 | 6.021117 | 61.87053 |
| 充分 发挥 | 191 | 0 | 2 | 6.590507 | 61.30822 |
# bigrams in 2017 report
ch17_col <- textstat_collocations(ch17_toks, size = 2)
knitr::kable(head(ch17_col, 10))| collocation | count | count_nested | length | lambda | z |
|---|---|---|---|---|---|
| 人民 群众 | 12 | 0 | 2 | 5.406843 | 12.89491 |
| 亿 元 | 14 | 0 | 2 | 8.302839 | 12.62184 |
| 调 控 | 11 | 0 | 2 | 7.593829 | 12.41301 |
| 政府 工作 | 9 | 0 | 2 | 4.710228 | 11.07990 |
| 深入 实施 | 8 | 0 | 2 | 5.018592 | 10.92455 |
| 党 中央 | 7 | 0 | 2 | 5.747235 | 10.90905 |
| 体制 改革 | 11 | 0 | 2 | 5.317394 | 10.53589 |
| 国内 生产 | 6 | 0 | 2 | 6.166877 | 10.48876 |
| 现代 化 | 8 | 0 | 2 | 5.706046 | 10.43500 |
| 基础 设施 | 7 | 0 | 2 | 7.549629 | 10.42514 |