Initial Setup

You can install the development version of anystyleR from Codeberg with:

# install.packages("devtools")
remotes::install_git("https://codeberg.org/agoutsmedt/anystyleR")

Before diving into anystyleR’s functionalities, ensure that Ruby and RubyGems are installed on your system. If you haven’t yet ventured into the Ruby world, no worries! Installation is straigthforward, and once set up, anystyleR will seamlessly manage the rest.

To automatically install anystyle and anystyle-cli Ruby gems, simply run:

library(anystyleR)
install_anystyle()

Utilizing anystyleR in your Research Workflow

anystyleR can be your go-to resource in various scenarios:

• Extracting references from multiple PDFs and converting them to .bib format.
• Building a structured data frame from references cited across several PDFs.
• Parsing a vector of references stored in an R object to extract detailed information.
• Converting a bibliography in a PDF from one citation style to another in word.

library(anystyleR)

# Locating the PDF documents bundled with the package
pdf_path <- system.file("extdata", package = "anystyleR")
pdf_files <- list.files(pdf_path, pattern = "pdf$", full.names = TRUE)

pdf_files

## [1] "C:/Users/goutsmedt/AppData/Local/R/win-library/4.3/anystyleR/extdata/example_doc_1.pdf"
## [2] "C:/Users/goutsmedt/AppData/Local/R/win-library/4.3/anystyleR/extdata/example_doc_2.pdf"
## [3] "C:/Users/goutsmedt/AppData/Local/R/win-library/4.3/anystyleR/extdata/example_doc_3.pdf"

# Leveraging pdftools to extract a page as an image
library(pdftools)
page_image <- pdftools::pdf_convert(pdf = pdf_files[1],
                                    format = 'png',
                                    pages = 22) 

## Converting page 22 to example_doc_1_22.png... done!

knitr::include_graphics(page_image)

find_ref(input = pdf_files[1],
         path = getwd(),
         output_format = "bib",
         overwrite = TRUE)

## [1] "anystyle --overwrite -f bib find C:/Users/goutsmedt/AppData/Local/R/win-library/4.3/anystyleR/extdata/example_doc_1.pdf C:/Users/goutsmedt/Documents/MEGAsync/Research/R/projets/website/content/en/post/2024-01-09-release-ranystyle-development"

# Locating and reading the .bib file corresponding to the first PDF
bib_file <- sub("pdf$", "bib",  basename(pdf_files[1]))
# Read the first 10 lines of the .bib file
bib_lines <- readLines(bib_file, n = 26)

# Print the lines
cat(bib_lines, sep = "\n")

## @article{arnold2023a,
##   author = {Arnold, M.},
##   date = {2023},
##   title = {ECB must do more to tackle inflation “Monster”, says christine lagarde’},
##   journal = {Financial Times}
## }
## @article{barro1983a,
##   author = {Barro, R.J. and Gordon, D.B.},
##   date = {1983},
##   title = {Rules, discretion and reputation in a model of monetary policy’},
##   volume = {12},
##   pages = {101–121},
##   url = {https://doi.org/10.1016/0304-3932(83)90051-X.},
##   doi = {10.1016/0304-3932(83)90051-X.},
##   journal = {Journal of Monetary Economics},
##   number = {1}
## }
## @article{b2009a,
##   author = {Béland, D.},
##   date = {2009},
##   title = {Ideas, institutions, and policy change’},
##   volume = {16},
##   pages = {701–718},
##   journal = {Journal of European Public Policy},
##   number = {5}
## }

# Extracting references from PDFs into a data frame
extracted_refs <- find_ref_to_df(input = pdf_files)

## [1] "anystyle -f json find C:/Users/goutsmedt/AppData/Local/R/win-library/4.3/anystyleR/extdata/example_doc_1.pdf "
## [1] "anystyle -f json find C:/Users/goutsmedt/AppData/Local/R/win-library/4.3/anystyleR/extdata/example_doc_2.pdf "
## [1] "anystyle -f json find C:/Users/goutsmedt/AppData/Local/R/win-library/4.3/anystyleR/extdata/example_doc_3.pdf "
## [1] "anystyle --overwrite -f ref find C:/Users/goutsmedt/AppData/Local/R/win-library/4.3/anystyleR/extdata/example_doc_1.pdf ./"
## [1] "anystyle --overwrite -f ref find C:/Users/goutsmedt/AppData/Local/R/win-library/4.3/anystyleR/extdata/example_doc_2.pdf ./"
## [1] "anystyle --overwrite -f ref find C:/Users/goutsmedt/AppData/Local/R/win-library/4.3/anystyleR/extdata/example_doc_3.pdf ./"

head(extracted_refs)

## # A tibble: 6 × 24
##   id_doc doc          id_ref author   title  year `container-title` volume pages
##    <int> <chr>        <chr>  <list>   <chr> <int> <chr>             <chr>  <chr>
## 1      1 example_doc… 1_1    <tibble> ECB …  2023 Financial Times   <NA>   <NA> 
## 2      1 example_doc… 1_2    <tibble> Rule…  1983 Journal of Monet… 12     101–…
## 3      1 example_doc… 1_3    <tibble> Idea…  2009 Journal of Europ… 16     701–…
## 4      1 example_doc… 1_4    <tibble> A st…  1999 Scottish Journal… 46     17–39
## 5      1 example_doc… 1_5    <tibble> Tech…  2018 International Po… 12     328–…
## 6      1 example_doc… 1_6    <tibble> Late…  2003 Journal of Machi… 3      <NA> 
## # ℹ 15 more variables: location <chr>, publisher <chr>, type <chr>, date <chr>,
## #   other_date <chr>, other_title <chr>, url <chr>, issue <chr>, doi <chr>,
## #   edition <chr>, genre <chr>, note <chr>, editor <chr>,
## #   `collection-title` <chr>, full_ref <chr>

# Preview the original full references for manual inspection and cleaning
head(extracted_refs$full_ref)

## [1] "Arnold, M. (2023). ECB must do more to tackle inflation “Monster”, says christine lagarde’. Financial Times"                                                                                       
## [2] "Barro, R. J., & Gordon, D. B. (1983). Rules, discretion and reputation in a model of monetary policy’. Journal of Monetary Economics, 12(1), 101–121. https://doi.org/10.1016/0304-3932(83)90051-X"
## [3] "Béland, D. (2009). Ideas, institutions, and policy change’. Journal of European Public Policy, 16(5), 701–718"                                                                                     
## [4] "Berger, H., & Haan, J. (1999). A state within the state? An event study on the bundesbank (1948–1973)’. Scottish Journal of Political Economy, 46(1), 17–39"                                       
## [5] "Best, J. (2018). Technocratic exceptionalism: Monetary policy and the fear of democracy’. International Political Sociology, 12(4), 328–345"                                                       
## [6] "Blei, D. M., Ng, A. Y., & Jordan, M. I. (2003). Latent Dirichlet Allocation’. Journal of Machine Learning Research, 3"

# Sample references in string format
references <- c("Thiemann, Matthias, Carolina Raquel Melches, and Edin Ibrocevic. 2021. “Measuring and Mitigating Systemic Risks: How the Forging of New Alliances Between Central Bank and Academic Economists Legitimize the Transnational Macroprudential Agenda.\" Review of international political economy 28 (6): 1433-1458",
                "Dietsch, Peter, François Claveau, and Clément Fontan. 2018. Do Central Banks Serve the People? New York: John Wiley & Sons",
                "Singleton, John. 2010. Central Banking in the Twentieth Century. Cambridge: Cambridge University Press",
                "Mudge, Stephanie L., and Antoine Vauchez. 2018. “Too Embedded to Fail.” Historical Social Research/Historische Sozialforschung 43(3): 248-273",
                "Lucas, Robert E. 1972. “Expectations and the Neutrality of Money”. Journal of Economic Theory 4(2): 103-124")

references

## [1] "Thiemann, Matthias, Carolina Raquel Melches, and Edin Ibrocevic. 2021. “Measuring and Mitigating Systemic Risks: How the Forging of New Alliances Between Central Bank and Academic Economists Legitimize the Transnational Macroprudential Agenda.\" Review of international political economy 28 (6): 1433-1458"
## [2] "Dietsch, Peter, François Claveau, and Clément Fontan. 2018. Do Central Banks Serve the People? New York: John Wiley & Sons"                                                                                                                                                                                       
## [3] "Singleton, John. 2010. Central Banking in the Twentieth Century. Cambridge: Cambridge University Press"                                                                                                                                                                                                           
## [4] "Mudge, Stephanie L., and Antoine Vauchez. 2018. “Too Embedded to Fail.” Historical Social Research/Historische Sozialforschung 43(3): 248-273"                                                                                                                                                                    
## [5] "Lucas, Robert E. 1972. “Expectations and the Neutrality of Money”. Journal of Economic Theory 4(2): 103-124"

# Parsing string references into a data frame
extracted_refs <- parse_ref_to_df(input = references)

## [1] "anystyle --no-overwrite -f json parse ref_to_parse.txt "

head(extracted_refs)

## # A tibble: 5 × 14
##   id_doc doc          id_ref author   title  year `container-title` volume pages
##    <int> <chr>        <chr>  <list>   <chr> <int> <chr>             <chr>  <chr>
## 1      1 ref_to_pars… 1_1    <tibble> Meas…  2021 Review of intern… 28     1433…
## 2      1 ref_to_pars… 1_2    <tibble> Do C…  2018 <NA>              <NA>   <NA> 
## 3      1 ref_to_pars… 1_3    <tibble> Cent…  2010 <NA>              <NA>   <NA> 
## 4      1 ref_to_pars… 1_4    <tibble> Too …  2018 Historical Socia… 43     248–…
## 5      1 ref_to_pars… 1_5    <tibble> Expe…  1972 Journal of Econo… 4      103–…
## # ℹ 5 more variables: location <chr>, publisher <chr>, type <chr>, date <chr>,
## #   issue <chr>

# Transforming PDF references into a Word document with a specific style
find_ref_to_word_bibliography(input = file.path(pdf_path, pdf_files[1]),
         csl_path = "path/to/your_style.csl")

Explore the Possibilities with anystyleR

The examples above are just a glimpse into the potential of anystyleR. I encourage you to delve deeper into each function and explore the package main vignette for a comprehensive understanding of what anystyleR can do.

As anystyleR is still in its developmental infancy, your feedback is invaluable. Please share your experiences, report any issues, suggest improvements, or propose new features. Your input will be instrumental in refining and enhancing anystyleR’s capabilities.

Credits

The anystyle parser, a core component of anystyleR, has been developed by a talented team: Alex Fenton, Sylvester Keil, Johannes Krtek and Ilja Srna. anystyle is under copyright: Copyright 2011-2018 Sylvester Keil. All rights reserved. See the Licence for details.

The anystyleR logo was created using DALL·E.

A look at the BoE Website

Let’s have a look, for instance, at the speeches of the BoE at https://www.bankofengland.co.uk/news/speeches.

Figure 1: BoE Speeches

We have a list of speeches that is relatively similar to what we had for the BIS, with a list of pages that you can change to the bottom of the webpage. However, the URL stays the same when you move to another page. I have not found any way to manipulate the URL in order to move from a page to another. But you could easily use RSelenium to click on the button to go to the next page.

Nonetheless, we can collect more information by clicking on each speech, as each speech has a dedicated webpage like this:

Figure 2: A Speech Webpage

This means that for each speech of the BoE’s policymakers, an URL exists, and thus we should be able to find this URL from the “sitemap”.²

Exploring the Sitemap

As explained in the first tutorial, it is important to tell the website who we are and to check the scraping rules on that site by reading the robot.txt.

root_url_BoE <- "https://www.bankofengland.co.uk"
user_agent <- "polite R package - used for https://github.com/agoutsmedt/central_bank_database (aurelien.goutsmedt[at]uclouvain.be)"
session <- bow(root_url_BoE,
               user_agent = user_agent)

session$robotstxt$text

## [robots.txt]
## --------------------------------------
## 
## User-agent: *
## Disallow: /boeapps/database/ShowChart.asp
## Disallow: /boeapps/database/_iadb-FromShowColumns.asp
## Disallow: /boeapps/iadb
## Disallow: /boeapps/titan
## Disallow: /error
## Disallow: /forms
## Disallow: /mfsd
## Disallow: /search
## Disallow: /test-folder
## Disallow: /-/media/boe/files/markets/benchmarks/sonia-statement-of-compliance-with-the-iosco-principles-for-financial-benchmarks.pdf
## 
## Sitemap: https://www.bankofengland.co.uk/_api/sitemap/getsitemap
## Host: www.bankofengland.co.uk

The robot.txt also indicates the URL of the sitemap:

sitemap <- session$robotstxt$sitemap$value

sitemap

## [1] "https://www.bankofengland.co.uk/_api/sitemap/getsitemap"

This webpage gathers a (very long) list of all the existing URLs of the website. You can try to have a look at it on your browser, but it takes a very long time to load. The easiest is to read it directly here using rvest.

xml_sitemap <- rvest::read_html(sitemap)

xml_sitemap %>% html_element(xpath = "body")

## {html_node}
## <body>
## [1] <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><url><loc>htt ...

We can see that it is the “loc” argument that interests us.

data_frame_sitemap <- tibble(hyperlink= xml_sitemap %>% 
                                   rvest::html_elements(xpath = ".//loc") %>% 
                                   rvest::html_text()) %>% 
  filter(hyperlink != root_url_BoE) # we exclude the base url that we don't need anymore

data_frame_sitemap

## # A tibble: 13,140 × 1
##    hyperlink                                                                    
##    <chr>                                                                        
##  1 https://www.bankofengland.co.uk/about                                        
##  2 https://www.bankofengland.co.uk/about/corporate-responsibility               
##  3 https://www.bankofengland.co.uk/about/corporate-responsibility/carbon-target…
##  4 https://www.bankofengland.co.uk/about/corporate-responsibility/charities-of-…
##  5 https://www.bankofengland.co.uk/about/get-involved                           
##  6 https://www.bankofengland.co.uk/about/get-involved/citizens-panels           
##  7 https://www.bankofengland.co.uk/about/get-involved/citizens-panels/cara-pete…
##  8 https://www.bankofengland.co.uk/about/get-involved/citizens-panels/emma-young
##  9 https://www.bankofengland.co.uk/about/get-involved/citizens-panels/insights-…
## 10 https://www.bankofengland.co.uk/about/get-involved/citizens-panels/nick-chen…
## # ℹ 13,130 more rows

We now have a list of 13140 URLs. What we need to do now is to understand what these URLs are about. We can find this by looking at what follows the base URLs:

data_frame_sitemap <- data_frame_sitemap %>% 
  mutate(category = str_remove(hyperlink, "https://www.bankofengland.co.uk/") %>% 
                     str_remove("/.*"))

unique(data_frame_sitemap$category) 

##  [1] "about"                               "accessibility"                      
##  [3] "agents-summary"                      "alternative-liquidity-facility"     
##  [5] "annual-report"                       "archive"                            
##  [7] "article"                             "asset-purchase-facility"            
##  [9] "bank-liabilities-survey"             "bank-overground"                    
## [11] "banking-services"                    "banknotes"                          
## [13] "careers"                             "ccbs"                               
## [15] "climate-change"                      "contact"                            
## [17] "coronavirus"                         "cost-of-living"                     
## [19] "credit-conditions-review"            "credit-conditions-survey"           
## [21] "decision-maker-panel"                "education"                          
## [23] "eu-withdrawal"                       "events"                             
## [25] "explainers"                          "external-mpc-discussion-paper"      
## [27] "faq"                                 "financial-policy-summary-and-record"
## [29] "financial-stability"                 "financial-stability-in-focus"       
## [31] "financial-stability-paper"           "financial-stability-report"         
## [33] "freedom-of-information"              "gold"                               
## [35] "independent-evaluation-office"       "inflation-attitudes-survey"         
## [37] "inflation-report"                    "legal"                              
## [39] "letter"                              "markets"                            
## [41] "minutes"                             "monetary-policy"                    
## [43] "monetary-policy-report"              "monetary-policy-summary-and-minutes"
## [45] "museum"                              "news"                               
## [47] "paper"                               "payment-and-settlement"             
## [49] "prudential-regulation"               "quarterly-bulletin"                 
## [51] "record"                              "report"                             
## [53] "research"                            "rss"                                
## [55] "sitemap"                             "speech"                             
## [57] "statement"                           "statistics"                         
## [59] "sterling-monetary-framework"         "sterling-money-market-survey"       
## [61] "stress-testing"                      "subscribe-to-emails"                
## [63] "systemic-risk-survey"                "the-digital-pound"                  
## [65] "vulnerability-disclosure-policy"     "weekly-report"                      
## [67] "whistleblowing"                      "working-paper"

We can now filter the URLs depending on what we want to scrape. For this time, let’s forget the speeches and say that we are interested in the “minutes”:

minutes_urls <- data_frame_sitemap %>% 
  filter(category == "minutes")

minutes_urls

## # A tibble: 669 × 2
##    hyperlink                                                            category
##    <chr>                                                                <chr>   
##  1 https://www.bankofengland.co.uk/minutes/1997/monetary-policy-commit… minutes 
##  2 https://www.bankofengland.co.uk/minutes/1997/monetary-policy-commit… minutes 
##  3 https://www.bankofengland.co.uk/minutes/1997/monetary-policy-commit… minutes 
##  4 https://www.bankofengland.co.uk/minutes/1997/monetary-policy-commit… minutes 
##  5 https://www.bankofengland.co.uk/minutes/1997/monetary-policy-commit… minutes 
##  6 https://www.bankofengland.co.uk/minutes/1997/monetary-policy-commit… minutes 
##  7 https://www.bankofengland.co.uk/minutes/1997/monetary-policy-commit… minutes 
##  8 https://www.bankofengland.co.uk/minutes/1998/monetary-policy-commit… minutes 
##  9 https://www.bankofengland.co.uk/minutes/1998/monetary-policy-commit… minutes 
## 10 https://www.bankofengland.co.uk/minutes/1998/monetary-policy-commit… minutes 
## # ℹ 659 more rows

Scraping Minutes Metadata

Thanks to the URLs of the minutes, we have the year of the minutes. Let’s focus on the minutes of 2023.

minutes_urls <- minutes_urls %>% 
  mutate(dates = str_extract(hyperlink, "(?<=/)\\d{4}(?=/)") %>% as.integer()) %>% 
  filter(dates == 2023)

Here is the first URL in our data:

Figure 3: A Minutes Webpage

Using rvest, we first extract the html code of the first webpage. You can see that the text of the minutes can be extracted directly from this webpage. We want to scrape the title of the document, its date, its summary, and the text content.

# first, we read the html
page <- minutes_urls %>% 
  slice(1) %>% 
  pull(hyperlink) %>% 
  read_html

# second, we extract the different information
extracted_title <-  page %>% 
  html_nodes("h1") %>% 
  html_text()
extracted_date <- page %>%  
  html_nodes(".published-date") %>% 
  html_text(trim = TRUE) %>% 
  str_extract("\\d+ [A-z]+ \\d+$") %>% 
  dmy()

# As the abstract and the text may gather several paragraphs, the object could be a vector of several strings. We thus save these strings as a list format (because we want one line per URL/document)
abstract <-  page %>% 
  html_nodes(".hero") %>% 
  html_text(trim= TRUE) %>% 
  list()
text <- page %>% 
  # We extract the titles and the text together
  html_nodes('.page-section .content-block h2 , .page-section .content-block h3, .page-section .content-block .page-content p')  %>% 
  html_text() %>% 
  list()

metadata <- tibble(title = extracted_title,
                   date = extracted_date,
                   abstract = abstract,
                   text = text)  

metadata

## # A tibble: 1 × 4
##   title                                                date       abstract text 
##   <chr>                                                <date>     <list>   <lis>
## 1 Minutes of the Meeting of the Court of Directors he… 2023-06-08 <chr>    <chr>

We can have a look at the first lines of the text extracted:

metadata %>%
  pull(text) %>% 
  unlist() %>% 
  .[1:30] %>%  
  str_c(collapse = "\n") %>% 
  cat()

## Present:
## David Roberts, Chair
## Andrew Bailey, Governor
## Ben Broadbent, Policy
## Sir Jon Cunliffe, Deputy Governor – Financial Stability
## Sir Dave Ramsden, Deputy Governor – Markets & Banking
## Sam Woods, Deputy Governor – Prudential Regulation
## Sabine Chalmers
## Lord Jitesh Gadhia
## Anne Glover
## Sir Ron Kalifa
## Diana Noble
## Frances O’Grady
## Tom Shropshire 
## In attendance:
## Ben Stimson, Chief Operating Officer
## Secretary:
## Sebastian Walsh
## 1. Conflicts, Minutes and Matters Arising
## There were no conflicts declared in relation to the present agenda.
## The minutes of the meeting held on 8 February 2023 were approved.
## The Chair noted that a number of steps had been introduced to support the effectiveness of Court meetings.  
## The Chair welcomed Ruth Smith to the meeting and noted that Ruth had been appointed as interim Secretary of the Bank.  
## 2. Governor’s Update
## The Governor updated Court on recent developments in the banking system.  
## The Governor reflected that recent weeks had seen two banks with UK operations fail and parts of the financial sector come under stress. Silicon Valley Bank (SVB) and its UK subsidiary failed on 10 March due to a deposit run following significant losses in the US parent. Court observed the speed of the run on SVB. Court noted the factors underlying this, including the extent to which social media now drives bank runs.
## The PRA’s approach to the UK operations of SVB ensured a greater degree of financial resilience, particularly the subsidiarisation policy. Over the weekend that SVB UK failed, the Bank conducted a successful resolution transaction – demonstrating the effective operation of the UK’s resolution regime. SVB UK’s capital instruments were written down, and the firm was sold to HSBC. The Bank’s actions protected both depositors and public funds.
## The Governor set out that the resolution plan for Credit Suisse was well-rehearsed by the UK, US and Swiss authorities. However, the Swiss authorities had not used the resolution plan. Instead, the Swiss conducted a going-concern sale underpinned by some public support, which in turn effected a full write-down of subordinated debt instruments, while equity was only partially written down. There had been some brief fallout in AT1 markets following this action, but swift communications from the UK and EU authorities on their approach to the creditor hierarchy in resolution had stemmed this.
## Though SVB and Credit Suisse had failed for different reasons, the events had taken place in close proximity to each other. As a result of this, investors perceived that the outlook for the global banking sector was uncertain. The Governor noted that more broadly the backdrop for these events was an uncertain outlook for global economic activity. The Bank was monitoring the financial system closely, particularly the impact of recent events on financial markets, UK banks and economic conditions. The Governor noted that UK banks are well capitalised, liquid and resilient to continue supporting households and businesses.
## Providing further detail about this work, Sam Woods explained that the FPC and the PRA had been closely examining developments across the financial system as a whole.

Now, we just need to build a loop to scrape the metadata for all the minutes. To avoid overloading the BoE website, we set a delay between navigating to the different URLs.

delay <- session$delay

We will also use the TryCatch() function to avoid breaking the loop in case of error (for instance when a page is not loaded correctly).

errors <- list()
metadata <- vector(mode = "list", length = nrow(minutes_urls))

  for(i in minutes_urls$hyperlink){ 
    tryCatch({
      page <- read_html(i)
      cat(paste("Now scraping URL: ", i, "\n"))
      Sys.sleep(delay)
      
      extracted_title <-  page %>% 
        html_nodes("h1") %>% 
        html_text()
      extracted_date <- page %>%  
        html_nodes(".published-date") %>% 
        html_text(trim = TRUE) %>% 
        str_extract("\\d+ [A-z]+ \\d+$") %>% 
        dmy()
      abstract <-  page %>% 
        html_nodes(".hero") %>% 
        html_text(trim= TRUE) %>% 
        list()
      text <- page %>% 
  # We extract the titles and the text together
      html_nodes('.page-section .content-block h2 , .page-section .content-block h3, .page-section .content-block .page-content p')  %>% 
      html_text() %>% 
        list()
      
      metadata[[i]] <- tibble(title = extracted_title,
                              url = i,
                              date = extracted_date,
                              abstract = abstract,
                              text = text)   
    }, error = function(e) {
      cat(paste("Problem with scraping URL: ", i, "\n"))
    errors[[i]] <- i
    })
  }

minutes_metadata <- bind_rows(metadata)

minutes_metadata

When all documents have their own webpage, the sitemap data becomes a powerful piece of information to facilitate your scraping. Here, it can be used to easily scrape all documents published by the Bank of England. However, the structure of the page is not exactly the same within a category (the text of the minutes is not available on the webpages before 2022 but you can scrape the URL of the PDF), and between categories (e.g., look at a speech or a working paper). Consequently, a more refined code is needed. So a more sophisticated script can be found here.

References

A look at the ECB Website

Let’s say we want to scrape the metadata of ECB’s “occasional papers” at “https://www.ecb.europa.eu/pub/research/occasional-papers/html/index.en.html”.

Figure 1: ECB’s occasional papers

We can see that there is 333 occasional papers on the ECB website. Let’s try to scrape them. First, we use polite to “bow” in front of the ECB website. We specify who we are in the user_agent parameter, in order for the webmaster to contact us in case of problem. Indeed, we are trying here to follow some scraping ethical guidelines (find more developments on this issue here). The bow() function allows us to look at permissions for scraping.

ecb_op_path <- "https://www.ecb.europa.eu/pub/research/occasional-papers/html/index.en.html"

session <- bow(ecb_op_path,
               user_agent = "polite R package - used for https://github.com/agoutsmedt/central_bank_database (aurelien.goutsmedt[at]uclouvain.be)")
session

## <polite session> https://www.ecb.europa.eu/pub/research/occasional-papers/html/index.en.html
##     User-agent: polite R package - used for https://github.com/agoutsmedt/central_bank_database (aurelien.goutsmedt[at]uclouvain.be)
##     robots.txt: 32 rules are defined for 1 bots
##    Crawl delay: 5 sec
##   The path is scrapable for this user-agent

By looking at the robots.txt file, we can see more in details which paths of the website are scrapable or not. Nothing about the publications path here!

cat(session$robotstxt$text)

## User-agent: *
## Sitemap: https://www.ecb.europa.eu/sitemap.xml
## Disallow: /*_content.bg.html$
## Disallow: /*_content.cs.html$
## Disallow: /*_content.da.html$
## Disallow: /*_content.de.html$
## Disallow: /*_content.el.html$
## Disallow: /*_content.en.html$
## Disallow: /*_content.es.html$
## Disallow: /*_content.et.html$
## Disallow: /*_content.fi.html$
## Disallow: /*_content.fr.html$
## Disallow: /*_content.ga.html$
## Disallow: /*_content.hr.html$
## Disallow: /*_content.hu.html$
## Disallow: /*_content.it.html$
## Disallow: /*_content.lt.html$
## Disallow: /*_content.lv.html$
## Disallow: /*_content.mt.html$
## Disallow: /*_content.nl.html$
## Disallow: /*_content.pl.html$
## Disallow: /*_content.pt.html$
## Disallow: /*_content.ro.html$
## Disallow: /*_content.sk.html$
## Disallow: /*_content.sl.html$
## Disallow: /*_content.sv.html$
## Disallow: /ecb/10ann/shared/movies/
## Disallow: /ecb/educational/pricestab/shared/movie/
## Disallow: /ecb/educational/shared/movies/
## Disallow: /paym/coll/assets/html/dla/EA/
## Disallow: /press/tvservices/webcast/shared/video/
## Disallow: /paym/coll/assets/html/dla/EA/
## Disallow: /events/conferences/shared/movie/
## Disallow: /euro/changeover/shared/data/
## Crawl-delay: 5

Let’s launch our bot.

remDr <- rsDriver(browser = "firefox",
                  port = 4444L,
                  chromever = NULL)
browser <- remDr[["client"]]
browser$navigate(ecb_op_path)
Sys.time(session$delay) # letting the page load
browser$findElement("css", "a.cross.linkButton.linkButtonLarge.floatRight.highlight-extra-light")$clickElement() # Refusing cookies

Scrolling down and opening supplementary menus

We can try something quickly: we extract the list of the occasional papers number (the “No. XXX” information just above the title).

categories <- browser$findElements("css selector", ".category") %>% 
  map_chr(., ~.$getElementText()[[1]]) 

tail(categories)

## [1] "No. 315" "No. 314" "No. 313" "No. 312" "No. 311" "No. 310"

We got something weird here. The last paper we got in the list is No. 310. But if we scroll all the way down to the page, we can see that the last paper should be “No. 1”.

Figure 2: ECB’s occasional papers

Let’s dig a bit into the html code of the webpage. In Firefox, you can go to the supplementary tools and open the “web development tools”. You can see that there is something called lazyload-container.

Figure 3: Digging into ECB’s website source code

As the number of occasional papers is quite big, for efficiency reasons, the website does not load the whole information. Consequently, if you want to scrape all the papers, you need to scroll down the page, and you need to scroll progressively to be sure that all the containers are loaded. The executeScript() function of RSelenium allows you to use some JavaScript code. Here, you don’t want to scroll horizontally, but you want to scroll vertically: you thus need to define the number of pixels by which you want to scroll down (let’s use 1600 to be sure that we will scroll down slowly). We will repeat this operation a certain number of time: as scrolling down by 1600 pixels is equivalent to scroll down by (a bit more than) five papers, we divide the number of papers by 5, to know the number of iterations of the scrolling down move.

scroll_height <- 1600
scroll_iteration <- str_extract(categories[1], "\\d+") %>% 
  as.numeric()/5

  for(i in 1:ceiling(scroll_iteration)) {
    browser$executeScript(glue("window.scrollBy(0,{scroll_height});"))
    Sys.sleep(0.3)
  }

Let’s check that now all the discussion papers are available:

categories <- browser$findElements("css selector", ".category") %>% 
  map_chr(., ~.$getElementText()[[1]]) 

length(categories)

## [1] 333

That’s all good. But we face a second issue: some important information are not visible yet. Indeed, by clicking on the “Details” button, you can access the abstract and the JEL codes of the paper.

Figure 4: The ‘details’ menu

But as long as the Details menu is closed, the information is not accessible. So we need to find all the buttons that allow to click on the “Details” menu. But be careful, we want only the “Details” button, and not other types of button. As in the former post, the ScrapeMate Firefox add-on is essential to find the right CSS selector.

Figure 5: Using ScrapeMate Beta

We can now click on all the “Details” buttons, and only the “Details” buttons:

  buttons <- browser$findElements("css selector", ".ecb-langSelector+ .accordion .header:nth-child(1) .title")
  for(i in seq_along(buttons)){
    buttons[[i]]$clickElement()
    Sys.sleep(0.4)
  }

Now, we have at our disposal all the information that is of interest to us. The remaining issue is that some information may be missing (or non-existent) for some papers (for instance, one paper has no author). So we extract the dates separately, and then we select the whole set of information for each paper (in the papers_whole_information object), and in this set, we extract in turn the title, the authors, the URL of the pdf, and the information in details.²

papers_whole_information <- browser$getPageSource()[[1]] %>% # we extract the whole info of each paper here (without the date)
  read_html() %>%
  html_elements("dd")

metadata <- tibble(
  category = categories,
  dates = browser$findElements("css selector", ".loaded > dt") %>% 
    map_chr(., ~.$getElementText()[[1]]),
  titles = papers_whole_information %>% 
    html_elements(".category+ .title a") %>% 
    html_text(),
  authors = papers_whole_information %>% 
    html_elements("ul") %>% 
    html_text2(),
  details = papers_whole_information %>% 
    html_elements(".ecb-langSelector+ .accordion .content-box:nth-child(2)") %>%
    html_text2(),
  url_pdf = papers_whole_information %>% 
    html_elements(".authors+ .ecb-langSelector .pdf") %>% 
    html_attr("href") 
)

head(metadata)

## # A tibble: 6 × 6
##   category dates             titles                      authors details url_pdf
##   <chr>    <chr>             <chr>                       <chr>   <chr>   <chr>  
## 1 No. 333  8 November 2023   Living in a world of disap… "Chiar… "Abstr… /pub/p…
## 2 No. 332  24 October 2023   The effects of high inflat… "Krzys… "Abstr… /pub/p…
## 3 No. 331  17 October 2023   The future of DAOs in fina… "Ellen… "Abstr… /pub/p…
## 4 No. 330  16 October 2023   Inflation, fiscal policy a… "Henri… "Abstr… /pub/p…
## 5 No. 329  20 September 2023 How usable are capital buf… "Georg… "Abstr… /pub/p…
## 6 No. 317  13 September 2023 Recent advances in the lit… "Rolan… "Abstr… /pub/p…

Two points of clarification here:

• the CSS selector must be selected carefully. For instance, a paper may have several “.title a”, because it gathers some complementary documents. Paper 209 is an example. So you only want the title after the category. Hence the selector “.category+ .title a”. That’s the same logic for the “details” menu: you don’t want what is in the Annex for instance. Also, you may have several pdf links, and you just want the pdf link below the authors, which is the pdf of the paper.
• the authors and details columns remain to be cleaned. Indeed, you can have several authors, and the details column gathers information about the abstract and JEL_codes, but also, in rare occasion, the “taxonomy” and the “network”. That’s why we use the html_text2() function of rvest: each type of information is separated by a linebreak (“”).

Let’s clean the details. You first separate the lines and get something like that:

details_cleaned <- metadata %>% 
  select(category, details) %>% 
  separate_longer_delim(details, "\n") %>% 
    filter(! details == "")

head(details_cleaned, 20)

## # A tibble: 20 × 2
##    category details                                                             
##    <chr>    <chr>                                                               
##  1 No. 333  Abstract                                                            
##  2 No. 333  The loss of biodiversity and the degradation of natural ecosystems …
##  3 No. 333  JEL Code                                                            
##  4 No. 333  C55 : Mathematical and Quantitative Methods→Econometric Modeling→Mo…
##  5 No. 333  G21 : Financial Economics→Financial Institutions and Services→Banks…
##  6 No. 333  G38 : Financial Economics→Corporate Finance and Governance→Governme…
##  7 No. 333  Q5 : Agricultural and Natural Resource Economics, Environmental and…
##  8 No. 332  Abstract                                                            
##  9 No. 332  The recent spike in inflation, unprecedented in the history of the …
## 10 No. 332  JEL Code                                                            
## 11 No. 332  C3 : Mathematical and Quantitative Methods→Multiple or Simultaneous…
## 12 No. 332  E3 : Macroeconomics and Monetary Economics→Prices, Business Fluctua…
## 13 No. 332  E6 : Macroeconomics and Monetary Economics→Macroeconomic Policy, Ma…
## 14 No. 331  Abstract                                                            
## 15 No. 331  Despite the crypto-market crash in the spring of 2022 and the colla…
## 16 No. 331  JEL Code                                                            
## 17 No. 331  F38 : International Economics→International Finance→International F…
## 18 No. 331  F39 : International Economics→International Finance→Other           
## 19 No. 331  G23 : Financial Economics→Financial Institutions and Services→Non-b…
## 20 No. 331  G32 : Financial Economics→Corporate Finance and Governance→Financin…

What you need to do is to extract the category of information (“Abstract|JEL Code|Taxonomy|Network”), and the text below represents the content of this category of information. You can then pivot the data to fill each column for the paper. Of course, most columns for taxonomy and network will be empty. As you have several JEL codes for each paper, pivot_wider() creates list columns for each type of information. But you can “unnest” the columns with single values (“Abstract”, “Taxonomy”, and “Network”)

details_cleaned <- details_cleaned %>% 
  mutate(type = str_extract(details, "^Abstract$|^JEL Code$|^Taxonomy$|^Network$")) %>% 
  fill(type, .direction = "down") %>% # we replace NA values in the type column by the information above which is non NA
  filter(! str_detect(details, "^Abstract$|^JEL Code$|^Taxonomy$|^Network$")) %>% # we remove the title rows
  pivot_wider(names_from = type, values_from = details) %>% 
  unnest(cols = c("Abstract", "Taxonomy", "Network")) 

head(details_cleaned)

## # A tibble: 6 × 5
##   category Abstract                                  `JEL Code` Network Taxonomy
##   <chr>    <chr>                                     <list>     <chr>   <chr>   
## 1 No. 333  The loss of biodiversity and the degrada… <chr [4]>  <NA>    <NA>    
## 2 No. 332  The recent spike in inflation, unprecede… <chr [3]>  <NA>    <NA>    
## 3 No. 331  Despite the crypto-market crash in the s… <chr [7]>  <NA>    <NA>    
## 4 No. 330  This paper analyses the distributional i… <chr [6]>  <NA>    <NA>    
## 5 No. 329  This paper analyses banks’ ability to … <chr [2]>  <NA>    <NA>    
## 6 No. 317  Large swings in cross-border capital flo… <chr [2]>  <NA>    <NA>

Let’s “zoom” to the JEL code column:

head(details_cleaned %>% select(category, `JEL Code`) %>%  unnest(`JEL Code`))

## # A tibble: 6 × 2
##   category `JEL Code`                                                           
##   <chr>    <chr>                                                                
## 1 No. 333  C55 : Mathematical and Quantitative Methods→Econometric Modeling→Mod…
## 2 No. 333  G21 : Financial Economics→Financial Institutions and Services→Banks,…
## 3 No. 333  G38 : Financial Economics→Corporate Finance and Governance→Governmen…
## 4 No. 333  Q5 : Agricultural and Natural Resource Economics, Environmental and …
## 5 No. 332  C3 : Mathematical and Quantitative Methods→Multiple or Simultaneous …
## 6 No. 332  E3 : Macroeconomics and Monetary Economics→Prices, Business Fluctuat…

Finally, you just need to merge the details cleaned with the metadata:

metadata_cleaned <- metadata %>% 
  select(-details) %>% 
  left_join(details_cleaned)

head(metadata_cleaned)

## # A tibble: 6 × 9
##   category dates     titles authors url_pdf Abstract `JEL Code` Network Taxonomy
##   <chr>    <chr>     <chr>  <chr>   <chr>   <chr>    <list>     <chr>   <chr>   
## 1 No. 333  8 Novemb… Livin… "Chiar… /pub/p… The los… <chr [4]>  <NA>    <NA>    
## 2 No. 332  24 Octob… The e… "Krzys… /pub/p… The rec… <chr [3]>  <NA>    <NA>    
## 3 No. 331  17 Octob… The f… "Ellen… /pub/p… Despite… <chr [7]>  <NA>    <NA>    
## 4 No. 330  16 Octob… Infla… "Henri… /pub/p… This pa… <chr [6]>  <NA>    <NA>    
## 5 No. 329  20 Septe… How u… "Georg… /pub/p… This pa… <chr [2]>  <NA>    <NA>    
## 6 No. 317  13 Septe… Recen… "Rolan… /pub/p… Large s… <chr [2]>  <NA>    <NA>

And you now have a nice table of metadata for ECB occasional papers! The post on the BIS explains then how to download the PDF, and how to run OCR if the text of the PDFs is not searchable. The code in this current post is also working for ECB’s “discussion papers” and “working papers”.

Another Method?

You can use the method above to scrape the metadata for the almost 3000 working papers of the ECB. But you can imagine that it takes a very long time to scroll down the page, and above all to open all the “Details” menu. RSelenium is wonderfully useful to interact with web browser, but sometimes, by digging into the source code of a webpage, you may find easier ways to extract relevant information.

And this is possible for the ECB research pages. We have already come across a clue to this earlier in this post. Indeed, let’s look again at the code related to the “lazyload-container”.

Figure 6: Digging into ECB’s website code

Actually it tells us that the website is loading some other pages, ending with “papers-2023.include.en.html”, “papers-2022.include.en.html”, etc., until “papers-1999.include.en.html”. Let’s try the first one: “https://www.ecb.europa.eu/pub/research/working-papers/html/papers-2023.include.en.html”

Figure 7: A simpler ECB page

So we can scrape quite easily the occasional papers for 2023:

url <- "https://www.ecb.europa.eu/pub/research/occasional-papers/html/papers-2023.include.en.html"

html_info <- read_html(url) %>%
  html_elements(xpath = "/html/body/dd")

metadata <- tibble(
  category = html_info %>% 
    html_elements(".category") %>% 
    html_text(),
  dates = read_html(url) %>%
    html_elements(".date") %>%
    html_text(),
  titles = html_info %>% 
    html_elements(".category+ .title a") %>% 
    html_text(),
  authors = html_info %>% 
    html_elements("ul") %>% 
    html_text2(),
  details = html_info %>% 
    html_elements(".ecb-langSelector+ .accordion .content-box:nth-child(2)") %>%
    html_text2(),
  url_pdf = html_info %>% 
    html_elements(".pdf") %>% 
    html_attr("href") 
)

head(metadata)

References

What is web scraping?

Web scraping is the process of extracting data from websites. It involves programmatically accessing a website’s content, extracting the data you want, and then saving it for further use. It involves going to a specific URL, retrieving the html content and extract the data you want from this content.

In general, you need to use web scraping when there is no API (Application Programming Interface) for the website data you want to access. An API is a structured and authorized way to access data provided by a website, from a system of queries (for instance, scopus provides an API to retrieve its bibliometric data).

Scraping data on a website implies various ethical and legal issues (for which I am not a specialist at all!). First, many websites have terms of service agreements that explicitly prohibit web scraping. Engaging in scraping without the website owner’s consent violates these terms and conditions. For instance, Linkedin prohibits scraping on its website. However, the “legality” of scraping Linkedin remains a complex matter. Before engaging in scraping, you need to know what are the terms and conditions, and know what are the risks if you violate them.

A second issue has to do with overloading servers. Web scraping can put a significant load on a website’s servers, potentially leading to decreased performance or downtime. Scraping without proper rate limiting and respect for a website’s capacity can be seen as unethical and disruptive.

Third, if you may be allowed to scrape a website, it does not mean that you can copy and distribute the materials collected. You should be aware of copyright laws and intellectual property rights when diffusing scraped content.

Some useful resources

When talking about web scraping with R, three R packages stand out (but we will only use two of them here):

• rvest (Wickham 2022): a package in the tidyverse spirit that helps to extract data from web pages. It works better with static web page (see below).
• RSelenium (Harrison 2022): this is an R binding for Selenium 2.0 Remote WebDriver, which allows you to automate the use of a browser. In other words, it enables you to write a script to open a web browser and interact with a website (yes, this is where scraping starts to get fun).
• polite (Perepolkin 2023): this package allows you to look at the robots.txt of a website to know what are the scraping rules for this website, and to avoid hammering the site with too many requests.

Depending on the website you want to scrap, using rvest may be sufficient (as it is the case, for instance, with Wikipedia). For more complicated website, you may need RSelenium, in combination with rvest or alone. Indeed, it depends if the web pages you want to scrape are static or dynamic web pages. As explained by Ivan Milanes’ useful blog post:

• Static Web Page: A Web page (HTML page) that contains the same information for all users. Although it may be periodically updated from time to time, it does not change with each user retrieval.
• Dynamic Web Page: A Web page that provides custom content for the user based on the results of a search or some other request. Also known as “dynamic HTML” or “dynamic content”, the “dynamic” term is used when referring to interactive Web pages created for each user.

The web page where central bankers’ speeches are gathered is a dynamic web page and so we will use RSelenium to scrape the speeches.¹

To complement this tutorial, you will find two other useful tutorials for scraping using rvest and RSelenium here and here. There is also a whole chapter dedicated to web scraping in the recent second edition of R for Data Science (Wickham, Çetinkaya-Rundel, and Grolemund 2023).

Scraping the BIS speeches

If we are interested in what central bankers are saying, we may want to study their speeches. Fortunately, most of them are stored on the BIS website there.

Figure 1: The BIS list of speeches

What we want is

• to extract the metadata of these speeches: the date of the speech, the title, its description and the speaker.
• to access to the whole text of the speech, that is stored in a pdf on the BIS website
• to extract the text from this PDF
• to be sure that the text is readable, which will be the case if the PDF is searchable (i.e. if characters are recognized).

This is all these things we will learn to do in this tutorial.

# pacman is a useful package to install missing packages and load them
if(! "pacman" %in% installed.packages()){
  install.packages("pacman", dependencies = TRUE)
}

pacman::p_load(tidyverse,
               RSelenium,
               polite,
               glue,
               here,
               pdftools,
               tesseract)

bis_data_path <- here(path.expand("~"), "data", "central_bank_database", "BIS") # Where data will go on my computer

Going on the BIS website and launching the bot

We use polite to “bow” in front of the BIS website. It allows us to look at permissions for scraping.

bis_website_path <- "https://www.bis.org/"

session <- bow(bis_website_path)
session

## <polite session> https://www.bis.org/
##     User-agent: polite R package
##     robots.txt: 15 rules are defined for 2 bots
##    Crawl delay: 5 sec
##   The path is scrapable for this user-agent

By looking at the robots.txt file, we can see more in details which paths of the website are scrapable or not.

cat(session$robotstxt$text)

## #Format is:
## #       User-agent: <name of spider>
## #       Disallow: <nothing> | <path>
## #-------------------------------------------
## 
## User-Agent: *
## Disallow: /dcms
## Disallow: /metrics/
## Disallow: /search/
## Disallow: /staff.htm
## Disallow: /embargo/
## Disallow: /app/
## Disallow: /goto.htm
## Disallow: /login
## #Disallow: /cbhub
## Disallow: /cbhub/goto.htm
## Disallow: /doclist/
## # Committee comment letters
## Disallow: /publ/bcbs*/
## Disallow: /bcbs/ca/
## Disallow: /bcbs/commentletters/
## Disallow: /*/publ/comments/
## # Hide the Basel Framework standards, only chapters should be indexed.
## Disallow: /basel_framework/standard/
## 
## Sitemap: https://www.bis.org/sitemap.xml

The list of speeches is at “https://www.bis.org/cbspeeches/index.htm”. This path is not among the “disallow” list, meaning that we can scrape this part of the website.

We want to make scraping easier, so we will display the maximum number of items per page, which is 25 here. We don’t want to scrape the whole list of speeches, so we will set a starting date: the ninth of October, 2023. By navigating towards the second page of the resulting list, you can now have an idea of what the URL looks like.

Figure 2: Filtering speeches on the BIS website

Thus, you can prepare a string with the URL to scrape, in which you can change the parameters.

scraping_path <- expr(glue("{bis_website_path}cbspeeches/index.htm?fromDate={day}%2F{month}%2F{year}&cbspeeches_page={page}&cbspeeches_page_length=25"))

Here, I could have used some simpler programming by using the paste0() function for instance, rather than using expr(). But I want to be able to change the parameters later, notably for the page number. For now, the path looks like this:

scraping_path

## glue("{bis_website_path}cbspeeches/index.htm?fromDate={day}%2F{month}%2F{year}&cbspeeches_page={page}&cbspeeches_page_length=25")

But then I can enter the parameters I want for the date (here, the ninth of October, 2023).

day <- "09"
month <- "10"
year <- 2023

And I can evaluate the expression with these parameters, also selecting a certain page. This will be useful later in a loop to navigate from one page to the next.

eval(scraping_path, envir = list(page = 3))

## https://www.bis.org/cbspeeches/index.htm?fromDate=09%2F10%2F2023&cbspeeches_page=3&cbspeeches_page_length=25

You can check that this URL leads you to the third page of the list of speeches starting the ninth of October, 2023 (with 25 results per page).

Once you have the structure of the URL, it means that you can change the parameters as you want to create the list of speeches you want. Of course, you can go on the BIS website and filter by the end date or the “Institutions” or the “Country” to see what the URL would look like.

Now we can start the funny part. Let’s launch our automated browser with RSelenium:

# Launch Selenium to go on the website of bis
driver <- rsDriver(browser = "firefox", # can also be "chrome"
                   chromever = NULL,
                   port = 4444L) 
remote_driver <- driver[["client"]]

Figure 3: Automatizing your web browser

Extracting the data page by page

What we want to do is to retrieve the data about each speech (the date, the title, its description, the speaker, and also its corresponding URL) page by page. This is something that we can do with a for loop. But we first need to know the number of pages. You can normally see it at the bottom right of the web page.

There could be different way to extract this information. Here, we will use the “CSS selector”. Using the CSS selector has two advantages:

• It has in general a simpler content than an XPATH
• It can be easily retrieved thanks to a Firefox add-on (which exists also for Chrome): ScrapeMate

Once you have set up the add-on, you can go to the URL we are working on and find the “CSS selector” for the number of pages:

Figure 4: Using scrapemate Firefox addon

You can use this CSS selector to extract the text thanks to RSelenium and after a short cleaning, you have the total number of pages:

remote_driver$navigate(eval(scraping_path, envir = list(page = 1)))
Sys.sleep(2) # Just to have time to load the page

nb_pages <- remote_driver$findElement("css selector", ".pageof")$getElementText() %>% # We just set i for searching the total number of pages
  pluck(1) %>% 
  str_remove_all("Page 1 of ") %>%
  as.integer()

nb_pages

## [1] 6

Ok. Now we can run the loop to retrieve all the data for the 6 pages. Here, we will conform to the delay set by the polite package: we will wait 5 seconds on each page, not to hammer the website with too many requests. As we need the web page to load properly to retrieve the data, it is also a way to take the time of loading the page to avoid missing data if your internet connection is too slow.

Here, you just have to use scrapemate to find the appropriate CSS selector for each information and extract the corresponding text. You can also see that by clicking on the title of the speech, you are navigating towards the dedicated web page of the corresponding speech. It means that to the title is associated a URL link, a “href” in html language. So you need to extract the “href” attribute associated to the title thanks to RSelenium.

metadata <- vector(mode = "list", length = nb_pages)

for(page in 1:nb_pages){
  remote_driver$navigate(eval(scraping_path))
  nod <- nod(session, eval(scraping_path)) # introducing to the new page
  Sys.sleep(session$delay) # using the delay time set by polite

  metadata[[page]] <- tibble(date = remote_driver$findElements("css selector", ".item_date") %>% 
                            map_chr(., ~.$getElementText()[[1]]),
                          info = remote_driver$findElements("css selector", ".item_date+ td") %>% 
                            map_chr(., ~.$getElementText()[[1]]),
                          url = remote_driver$findElements("css selector", ".dark") %>% 
                            map_chr(., ~.$getElementAttribute("href")[[1]])) 
}

metadata <- bind_rows(metadata) %>% 
  separate(info, c("title", "description", "speaker"), "\n")

head(metadata)

## # A tibble: 6 × 5
##   date        title                                    description speaker url  
##   <chr>       <chr>                                    <chr>       <chr>   <chr>
## 1 27 Nov 2023 "Ahmet Ismaili: Welcoming remarks - Wor… "Welcoming… by Ahm… http…
## 2 27 Nov 2023 "Ahmet Ismaili: Creating values – a gat… "Introduct… by Ahm… http…
## 3 22 Nov 2023 "Christine Lagarde: Monetary policy in … "Speech by… by Chr… http…
## 4 22 Nov 2023 "Sabine Mauderer: Exposing and acting o… "Speech by… by Sab… http…
## 5 21 Nov 2023 "Jon Cunliffe: Money and payments - a \… "Speech by… by Jon… http…
## 6 21 Nov 2023 "Shaktikanta Das: Emerging India - a la… "Keynote s… by Sha… http…

The findElements() function of RSelenium finds all the corresponding elements (here 25) and stores them in a list. The map_chr() allows you to get the text of each element one by one and stores the result in a vector.

If we want to be sure that we have retrieved the data of all the speeches, we can compare the length of our data frame, with the number of items that was indicated just above the first speech.

# We just check that we have the correct number of speeches
nb_items <- remote_driver$findElement("css selector", ".listitems")$getElementText() %>% # We just set i for searching the total number of pages
  pluck(1) %>%
  str_remove_all("[\\sitems]") %>% 
  as.integer()
if(nb_items == nrow(metadata)){
  cat("You have extracted the metadata of all speeches")
} else{
  cat("There is a problem. You have probably missed some speeches.")
}

## You have extracted the metadata of all speeches

pdf_to_download <- metadata %>% 
    mutate(pdf_name = str_extract(url, "r\\d{6}[a-z]"),
           pdf_name = str_c(pdf_name, ".pdf")) %>% 
    pull(pdf_name) 

pdf_to_download

##   [1] "r231127c.pdf" "r231127d.pdf" "r231122b.pdf" "r231122a.pdf" "r231121a.pdf"
##   [6] "r231115e.pdf" "r231121i.pdf" "r231121h.pdf" "r231121g.pdf" "r231121f.pdf"
##  [11] "r231121e.pdf" "r231121d.pdf" "r231121c.pdf" "r231121b.pdf" "r231120j.pdf"
##  [16] "r231120h.pdf" "r231120i.pdf" "r231114a.pdf" "r231114b.pdf" "r231115r.pdf"
##  [21] "r231115s.pdf" "r231114c.pdf" "r231115t.pdf" "r231115u.pdf" "r231120g.pdf"
##  [26] "r231120f.pdf" "r231120e.pdf" "r231120d.pdf" "r231120c.pdf" "r231120b.pdf"
##  [31] "r231120a.pdf" "r231113s.pdf" "r231115q.pdf" "r231115o.pdf" "r231115n.pdf"
##  [36] "r231115m.pdf" "r231115j.pdf" "r231115i.pdf" "r231115f.pdf" "r231115h.pdf"
##  [41] "r231115g.pdf" "r231114d.pdf" "r231115d.pdf" "r231115p.pdf" "r231113n.pdf"
##  [46] "r231115a.pdf" "r231113m.pdf" "r231115c.pdf" "r231114e.pdf" "r231114f.pdf"
##  [51] "r231113t.pdf" "r231113o.pdf" "r231113q.pdf" "r231113x.pdf" "r231113w.pdf"
##  [56] "r231113v.pdf" "r231113u.pdf" "r231113p.pdf" "r231113r.pdf" "r231113k.pdf"
##  [61] "r231113j.pdf" "r231113i.pdf" "r231113h.pdf" "r231113g.pdf" "r231113f.pdf"
##  [66] "r231113e.pdf" "r231113d.pdf" "r231113c.pdf" "r231113b.pdf" "r231113l.pdf"
##  [71] "r231113a.pdf" "r231110a.pdf" "r231108c.pdf" "r231108b.pdf" "r231108a.pdf"
##  [76] "r231107c.pdf" "r231107b.pdf" "r231107a.pdf" "r231102a.pdf" "r231027b.pdf"
##  [81] "r231115l.pdf" "r231026l.pdf" "r231026j.pdf" "r231026q.pdf" "r231026g.pdf"
##  [86] "r231026f.pdf" "r231026e.pdf" "r231026m.pdf" "r231026d.pdf" "r231026c.pdf"
##  [91] "r231026b.pdf" "r231026a.pdf" "r231023h.pdf" "r231023g.pdf" "r231023e.pdf"
##  [96] "r231023c.pdf" "r231023b.pdf" "r231023a.pdf" "r231020f.pdf" "r231020e.pdf"
## [101] "r231020d.pdf" "r231020c.pdf" "r231020b.pdf" "r231020a.pdf" "r231019g.pdf"
## [106] "r231019e.pdf" "r231019d.pdf" "r231019c.pdf" "r231019b.pdf" "r231019a.pdf"
## [111] "r231018f.pdf" "r231018e.pdf" "r231018d.pdf" "r231018c.pdf" "r231018b.pdf"
## [116] "r231018a.pdf" "r231017e.pdf" "r231017d.pdf" "r231017c.pdf" "r231017b.pdf"
## [121] "r231012a.pdf" "r231017a.pdf" "r231011d.pdf" "r231012b.pdf" "r231011g.pdf"
## [126] "r231011f.pdf" "r231011e.pdf" "r231011a.pdf" "r231011b.pdf" "r231010h.pdf"
## [131] "r231010g.pdf" "r231010f.pdf" "r231010e.pdf" "r231010d.pdf" "r231010c.pdf"
## [136] "r231010b.pdf" "r231010a.pdf" "r231115v.pdf"

polite_download <- function(domain, url, ...){
  bow(domain) %>% 
    nod(glue(url)) %>% 
    rip(...)  
}

tryCatch(
   {
     walk(pdf_to_download, ~polite_download(domain = bis_website_path,
                                            url = glue("review/{.}"),
                                            path = here(bis_data_path, "pdf"),))
   },
   error = function(e) {
     print(glue("No pdf exist"))
   }
  )

Extracting speeches text

Now that we have collected the PDFs, we want to extract the text within these PDFs. We will use the package pdftools for that. Here is a function that take the list of PDFs extracted from the metadata, extract the text, and stores it in a table with the name of the file and the corresponding page of the text (indeed, pdf_text() extract the text page by page). Here, we handle error in case we did not find the PDF: we don’t want the extraction of text to stop, but we want to remember that a PDF is missing.

extracting_text <- function(file, path) {
  tryCatch(
    {  
  message(glue("Extracting text from {file}"))
  text_data <- pdf_text(here(path, file)) %>%
    as_tibble() %>% 
    mutate(page = 1:n(),
           file = file) %>% 
    rename(text = value)
    },
  error = function(e) {
    print(glue("No pdf exist"))
    text_data <- tibble(text = NA,
                        page = 1,
                        file = file)
  }
  )
}

We can run the function for each speech we have in our dataset.

text_data <- map(pdf_to_download, ~extracting_text(., here(bis_data_path, "pdf"))) %>% 
  bind_rows()

## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist
## No pdf exist

head(text_data)

## # A tibble: 6 × 3
##   text                                                                page file 
##   <chr>                                                              <dbl> <chr>
## 1 "Ahmet Ismaili: Welcoming remarks - World Savings Day\nWelcoming …     1 r231…
## 2 "Young people should set clear financial goals for themselves and…     2 r231…
## 3 "Ahmet Ismaili: Creating values – a gateway for the future\nIntro…     1 r231…
## 4 "Therefore, in CBK's supervisory role, the external auditor profe…     2 r231…
## 5 "3/3   BIS - Central bankers' speeches\n"                              3 r231…
## 6 "Christine Lagarde: Monetary policy in the euro area - attentive …     1 r231…

Managing OCR problems

Sometimes, PDFs have problems of OCR. The PDFs are no searchable and so the text cannot be extracted properly (or rather, it is unreadable). It happens also with recent speeches. Therefore, we need to find the PDFs that may encounter such problems. We count the number of words in each page, and retrieve the PDFs for which the number of words is low in each page.

pdf_without_ocr <- text_data %>% 
  mutate(test_ocr = str_count(text, "[a-z]+")) %>% # Count the number of words
  mutate(non_ocr_document = all(test_ocr < 250), .by = file) %>% 
  filter(non_ocr_document == TRUE) %>% 
  distinct(file) %>% 
  pull()

pdf_without_ocr

##  [1] "r231122a.pdf" "r231121h.pdf" "r231121g.pdf" "r231121d.pdf" "r231110a.pdf"
##  [6] "r231026f.pdf" "r231026c.pdf" "r231023h.pdf" "r231023g.pdf" "r231017d.pdf"
## [11] "r231017b.pdf" "r231011g.pdf" "r231010g.pdf" "r231010e.pdf" "r231010d.pdf"
## [16] "r231010c.pdf" "r231010b.pdf"

Setting the threshold at 250 words is a pure rule of thumb: it allows having only true positive, but it does not prevent for having false negative (missing documents with OCR problem). It should be possible to find a better method to assess which PDF have no OCR.

Here is an example of a problematic PDF:

text_data %>% 
  filter(file %in% pdf_without_ocr[1])

## # A tibble: 6 × 3
##   text                                                                page file 
##   <chr>                                                              <dbl> <chr>
## 1 "012345637839 \u0011 3 7839843 \u0011\u000e433\u000f\u00103783984…     1 r231…
## 2 "01ÿ34567567ÿ85996ÿ45 ÿ\u000e\u000fÿ56\u000fÿ\u000f8ÿ\u00106ÿ\u00…     2 r231…
## 3 "0123456ÿ89252 49ÿ2 8ÿ\u000e8ÿ\u000f2ÿ94\u0010\u000f8ÿ9\u0011\u00…     3 r231…
## 4 "01ÿ3456ÿ17389ÿ83ÿ 8ÿ454543ÿ348ÿ57188\u000e51 ÿ\u000f7\u000e\u001…     4 r231…
## 5 "0123ÿ56ÿ718ÿ319\n      ÿ\u000e\u000f9\u00103\u00112\u0012ÿ\u0013…     5 r231…
## 6 "0ÿ2345678934\n    \u000e\u000fÿ\u0011 ÿ\u0012\u000e\u0011\u0013\…     6 r231…

It seems ok when you look at it, but going to the web page allows us to see that there are some problem of selection in the text and so that the OCR is not adequate.

Figure 5: PDF without adequate OCR

We will now use the tesseract package, that runs OCR. It first converts each page of the PDF into a picture (a PNG), then proceeds to OCR and extracts the text.

new_text_ocr <- vector(mode = "list", length = length(pdf_without_ocr))

for(i in 1:length(pdf_without_ocr)){
  text <- tesseract::ocr(here(bis_data_path, "pdf", pdf_without_ocr[i]), engine = tesseract::tesseract("eng"))
  text <- tibble(text) %>% 
    mutate(text = str_remove(text, ".+(=?\\\n)"),
           page = 1:n(),
           file = str_remove(pdf_without_ocr[i], "\\.pdf"))
  new_text_ocr[[i]] <- text
}

new_text_ocr <- bind_rows(new_text_ocr)

## Converting page 1 to r231122a_1.png... done!
## Converting page 2 to r231122a_2.png... done!
## Converting page 3 to r231122a_3.png... done!
## Converting page 4 to r231122a_4.png... done!
## Converting page 5 to r231122a_5.png... done!
## Converting page 6 to r231122a_6.png... done!

You can see that the text is now readable:

new_text_ocr

## # A tibble: 6 × 3
##   text                                                                page file 
##   <chr>                                                              <int> <chr>
## 1 "Exposing and acting on risks in a fast-changing\nworld\nEIOPA An…     1 r231…
## 2 "that might be emerging, we might also be able to open up new hor…     2 r231…
## 3 "industry. Recent earnings releases by large insurers show that h…     3 r231…
## 4 "Financial System (\n\nNGFS (Central Banks and Supervisors Networ…     4 r231…
## 5 "\nNGFS (Central Banks and Supervisors Network for Greening the F…     5 r231…
## 6 "\nLadies and gentlemen,\n\nTo come back to the animal-based meta…     6 r231…

Now you just have to integrate the now recognized text with the rest of the data. A last small step is to remove all the PNGs you have created by using tesseract.

# We remove all the .png we have created
png_to_remove <- list.files()[str_which(list.files(), "_\\d+\\.png")]
file.remove(png_to_remove)

And that’s it for the tutorial. You now have a data frame with the metadata of the speeches, and a data frame with the text, page by page for each speech.

If you want to scrap BIS speeches on a regular basis without scraping again what you have already collected, you may have a look at this script here, which is part of a project to collect central banks’ documents. You will also find in it some function to clean BIS data.

References

Search on Dimensions website

So let’s begin with the same search than in the last post: publications using Dynamic Stochastic General Equilibrium model.² Like with Scopus, we search for “DSGE” or “Dynamic Stochastic General Equilibrium”. A difference here is that we can choose between search in “full data” (meaning searching also in the full-text) or in “Title and abstract” (see Figure 1). Doing the first search, we get 27 693 results on February 4, 2022. The fact that we get close to 100 results before the 1990s (when the label DSGE began to be used) means that we could have a lot of false positive. It should be explored more in-depth. But for now and for this tutorial, we will opt for a more secure search, by focusing on titles and abstracts.

Figure 1: Our search on Dimensions app

We obtain 4106 results on February 4, 2022. That is quite much than the 2633 results of our Scopus search in the last post. But I think that it can be partly explain by the fact that Dimensions integrate “preprints” and we have 1532 preprints in our results.

To download the result, click on “Save / Export” on the right of the search bar, and then “Export results”. Two types of export are of interest for us: “Export full record” and “Export for bibliometric mapping”. In the first case, we can export only 500 results against 2500 for the second one (see Figure 2). This is not obvious, but the second option allows you to export most metadata, including affiliations and the references cited. What do we lose in comparison to the first export? The publication type (article, preprints, etc…), acknowledgements (that can be useful; see for instance Paul-Hus et al. 2017) and categorization of publications. That is not so much. Besides, it means that, by using the second option, you can extract more results than in Scopus (limited to 2000).

Figure 2: Choosing the export format

Another advantage of Dimensions over Scopus is that the data exported are quite easier to clean. So let’s load the needed packages and start the cleaning.

# packages
package_list <- c(
  "here", # use for paths creation
  "tidyverse",
  "janitor", # useful functions for cleaning imported data
  "biblionetwork", # creating edges
  "tidygraph", # for creating networks
  "ggraph" # plotting networks
)
for (p in package_list) {
  if (p %in% installed.packages() == FALSE) {
    install.packages(p, dependencies = TRUE)
  }
  library(p, character.only = TRUE)
}

github_list <- c(
  "agoutsmedt/networkflow", # manipulating network
  "ParkerICI/vite" # needed for the spatialisation of the network
)
for (p in github_list) {
  if (gsub(".*/", "", p) %in% installed.packages() == FALSE) {
    devtools::install_github(p)
  }
  library(gsub(".*/", "", p), character.only = TRUE)
}

# paths
data_path <- here(path.expand("~"),
                  "data",
                  "tuto_biblio_dsge")
dimensions_path <- here(data_path, 
                    "dimensions")

Cleaning Dimensions data

Here are the data, extracted in two steps as we have more than 2500 items:

dimensions_data_1 <- read_csv(here(dimensions_path,
                                "dimensions_dsge_data_2015-2022.csv"),
                           skip = 1)
dimensions_data_2 <- read_csv(here(dimensions_path,
                                            "dimensions_dsge_data_1996-2014.csv"),
                                       skip = 1)

dimensions_data <- rbind(dimensions_data_1,
                                  dimensions_data_2) %>% 
  clean_names() # cleaning column names with janitor to transform them

dimensions_data

## # A tibble: 4,097 x 15
##    publication_id doi      title abstract source_title_an~ pub_year volume issue
##    <chr>          <chr>    <chr> <chr>    <chr>               <dbl> <chr>  <chr>
##  1 pub.1126055399 10.1016~ Coro~ •We pro~ Annals of Touri~     2020 83     <NA> 
##  2 pub.1143049966 10.1016~ The ~ This pa~ China Economic ~     2021 71     <NA> 
##  3 pub.1126582656 10.1073~ Conf~ We inve~ Proceedings of ~     2020 117    17   
##  4 pub.1133125128 10.3390~ To F~ The ong~ Entropy              2020 22     12   
##  5 pub.1136926910 10.1002~ Impa~ Coronav~ Managerial and ~     2021 42     7    
##  6 pub.1124254349 10.3390~ Macr~ Assessm~ Entropy              2020 22     2    
##  7 pub.1120723604 10.1016~ Carb~ In rece~ The Science of ~     2019 697    <NA> 
##  8 pub.1144700962 10.3390~ Immu~ The COV~ Entropy              2022 24     1    
##  9 pub.1141415103 10.1016~ Ener~ Since t~ Structural Chan~     2021 59     <NA> 
## 10 pub.1128536448 10.1177~ Does~ This ar~ Journal of Hosp~     2020 45     1    
## # ... with 4,087 more rows, and 7 more variables: pagination <chr>,
## #   authors <chr>, authors_affiliations_name_of_research_organization <chr>,
## #   authors_affiliations_country_of_research_organization <chr>,
## #   dimensions_url <chr>, times_cited <dbl>, cited_references <chr>

We don’t need to give an ID to our documents as there is already a publications_id column.³

duplicated_articles <- dimensions_data %>%
  add_count(title) %>% 
  filter(n > 1) %>% 
  arrange(title, cited_references)

to_keep <- duplicated_articles %>% 
  group_by(title) %>% 
  slice_head(n = 1)

to_remove <- duplicated_articles %>% 
  filter(! publication_id %in% to_keep$publication_id)

dimensions_data <- dimensions_data %>% 
  filter(! publication_id %in% to_remove$publication_id)

Cleaning affiliations

The goal now is to create a table which associates each document to authors’ affiliation (column authors_affiliations_name_of_research_organization) and the country of the affiliation (column authors_affiliations_country_of_research_organization). We cannot associate affiliations directly to authors as documents can have more authors than affiliations, and conversely.

dimensions_affiliations <- dimensions_data %>% 
  select(publication_id,
         authors_affiliations_name_of_research_organization,
         authors_affiliations_country_of_research_organization) %>% 
  separate_rows(starts_with("authors"), sep = "; ")

knitr::kable(head(dimensions_affiliations))

publication_id	authors_affiliations_name_of_research_organization	authors_affiliations_country_of_research_organization
pub.1126055399	Temple University	United States
pub.1126055399	Jiangxi University of Finance and Economics	China
pub.1126055399	University of Connecticut	United States
pub.1143049966	Beijing Normal University	China
pub.1126582656	Laboratory of Theoretical Physics of Condensed Matter	France
pub.1126582656	École Polytechnique	France

We can now compare the country of affiliations for the dimensions document and for the Scopus affiliation cleaned in the previous post. We first bind together affiliations of DSGE articles from Scopus and Dimensions.

scopus_affiliations <- read_rds(here(data_path, 
                                     "scopus", 
                                     "data_cleaned", 
                                     "scopus_affiliations.rds")) %>% 
  mutate(data = "scopus") %>% 
  select(-authors)

dimensions_affiliations <- dimensions_affiliations %>% 
  mutate(data = "dimensions")

affiliations <- dimensions_affiliations %>% 
  rename("affiliations" = authors_affiliations_name_of_research_organization,
         "country" = authors_affiliations_country_of_research_organization,
         "citing_id" = publication_id) %>% 
  bind_rows(scopus_affiliations)

In Figure @ref(fig:country_affiliations), we can observe that the ranking is almost the same at least for the first 8 countries. Nonetheless, while we have more documents in Dimensions(3856) than in Scopus (2565), we have a relatively equivalent number of affiliations in both corpus: 5502 for Dimensions against 5461 for Scopus. Besides, we count more NA values (1132) than affiliations from United States (1112) in Dimensions, meaning that a lot of information is missing.

affiliations %>% 
  filter(!is.na(country)) %>% 
  group_by(data) %>% 
  count(country) %>% 
  mutate(proportion = n/sum(n)) %>% 
  slice_max(order_by = proportion, n = 10, with_ties = FALSE) %>% 
  mutate(country = tidytext::reorder_within(country, proportion, data)) %>% # see https://juliasilge.com/blog/reorder-within/ for more info
  ggplot(aes(proportion, country, fill = data)) +
  geom_col(show.legend = FALSE) +
  facet_wrap(~data, scales = "free_y") + 
  tidytext::scale_y_reordered() +
  labs(x = "Proportion of the Country in all affiliations (excluding NA)") +
  theme_classic(base_size = 16)

Figure 3: Top countries for affiliations in Scopus and Dimensions

Cleaning references

Cleaning the references is quite easy for Dimensions:

• you have a clear separator between the references of each citing document: a semi-colon before a square bracket;
• you have the same information for each references (see column_names) even if the information is lacking, and you have a simple separator between information: |.

references_extract <- dimensions_data %>% 
  filter(! is.na(cited_references)) %>% 
  rename("citing_id" = publication_id) %>% # because the "publication_id" column is also the identifier of the reference
  select(citing_id, cited_references) %>% 
  separate_rows(cited_references, sep = ";(?=\\[)") %>%
  as_tibble

column_names <- c("authors",
                  "author_id",
                  "source",
                  "year",
                  "volume",
                  "issue",
                  "pagination",
                  "doi",
                  "publication_id",
                  "times_cited")

dimensions_direct_citation <- references_extract %>% 
  separate(col = cited_references, into = column_names, sep = "\\|")

knitr::kable(head(dimensions_direct_citation, n = 5))

citing_id	authors	author_id	source	year	volume	issue	pagination	doi	publication_id	times_cited
pub.1126055399	[Yagihashi, Takeshi; Du, Juan]	[ur.07662443654.76; ur.013363010625.14]	Economic Inquiry	2015	53	3	1556-1579	10.1111/ecin.12204	pub.1003172294	5
pub.1126055399	[Yan, Qi; Zhang, Hanqin Qiu]	[; ]	Asia Pacific Journal of Tourism Research	2012	17	5	534-550	10.1080/10941665.2011.627929	pub.1052463830	5
pub.1126055399	[Barro, Robert J]	[]	The Quarterly Journal of Economics	2006	121	3	823-866	10.1162/qjec.121.3.823	pub.1063352797	1133
pub.1126055399	[Hall, R. E.; Jones, C. I.]	[; ]	The Quarterly Journal of Economics	2007	122	1	39-72	10.1162/qjec.122.1.39	pub.1063352822	450
pub.1126055399	[Gourio, François]	[]	American Economic Review	2012	102	6	2734-2766	10.1257/aer.102.6.2734	pub.1064525837	372

Another advantage is that, as for Scopus API (but not for data extracted on Scopus website), references already have an identifier.⁷ It means that you don’t have to match the citations together to find which documents are citing the same references (what we had to do here). Thus, you can easily extract the list of the distinct references cited in your corpus, and it will make easier the building of a co-citation network later.

dimensions_references <- dimensions_direct_citation %>% 
  distinct(publication_id, .keep_all = TRUE) %>% 
  select(-citing_id)

knitr::kable(tail(dimensions_references, n = 5))

authors	author_id	source	year	volume	issue	pagination	doi	publication_id	times_cited
[Barro, Robert J.; Grossman, Herschel I.]	[ur.01313542524.54; ur.015506435407.81]	The Review of Economic Studies	1974	41	1	87	10.2307/2296401	pub.1069867975	65
[Benassy, Jean-Pascal]	[]	Scandinavian Journal of Economics	1977	79	2	147	10.2307/3439505	pub.1070324476	41
[Delli Gatti, Domenico; Gallegati, Marco; Minsky, Hyman P.]	[ur.011273107226.57; ; ur.015265626417.37]	SSRN Electronic Journal	1998				10.2139/ssrn.125848	pub.1102184434	20
[Beker, Victor A.]	[]	SSRN Electronic Journal	2005				10.2139/ssrn.839307	pub.1102214017	5
[Beker, Victor A.]	[]	SSRN Electronic Journal	2010				10.2139/ssrn.1730222	pub.1102299065	7

However, there is one problem with Dimensions ID: I suppose that Dimensions is able to match references together thanks to DOI that are often present in references data (more than in Scopus references data). The problem is that some references (in general books and chapters in books) lack many (actually almost all) information, like authors and publishing source. It means that we are not able to know what it is, if not by entering the DOI in a web browser. We will encounter this problem again when building the network.

As for affiliations country, we can compare the most cited references in Dimensions and in Scopus. We first extract the top 10 for Scopus and Dimensions:

scopus_direct_citation <- read_rds(here(data_path,
                                        "scopus",
                                        "data_cleaned",
                                        "scopus_manual_direct_citation.rds"))
scopus_references <- read_rds(here(data_path,
                                   "scopus",
                                   "data_cleaned",
                                   "scopus_manual_references.rds"))

scopus_top_ref <- scopus_direct_citation %>% 
  add_count(new_id_ref) %>% 
  mutate(proportion = n/n()) %>% 
  select(new_id_ref, proportion) %>% 
  unique() %>% 
  slice_max(proportion, n = 10) %>%
  left_join(select(scopus_references, new_id_ref, references)) %>% 
  select(references, proportion) %>% 
  mutate(data = "scopus",
         references = str_extract(references, ".*\\(\\d{4}\\)")) # cleaning for plotting

dimensions_references <- dimensions_references %>% 
  mutate(references = paste0(authors, ", ", year, ", ", source))

dimensions_top_ref <- dimensions_direct_citation %>% 
  add_count(publication_id) %>% 
  mutate(proportion = n/n()) %>% 
  select(publication_id, proportion) %>% 
  unique() %>% 
  slice_max(proportion, n = 10) %>%
  left_join(select(dimensions_references, publication_id, references)) %>% 
  select(references, proportion) %>% 
  mutate(data = "dimensions")

We can see in Figure 4 that the 10 most cited references are the same, only the ranking is changing after the fourth position

dimensions_top_ref %>% 
  bind_rows(scopus_top_ref) %>% 
  mutate(references = str_wrap(references, 35)) %>% 
  ggplot(aes(proportion, fct_reorder(references, proportion), fill = data)) +
  geom_col(show.legend = FALSE) +
  facet_wrap(~data, scales = "free") + 
  tidytext::scale_y_reordered() +
  labs(x = "Proportion on all citations") +
  theme_classic(base_size = 13)

Figure 4: Top references of DSGE documents in Scopus and Dimensions

Building co-citation network

Here, we follow what we have done for Scopus here. We use the biblio_cocitation function of the biblionetwork package to build the list of edges between our nodes (i.e. the references). The edge between two nodes is weighted depending of the total number of times each reference has been cited in the whole Dimensions DSGE corpus (see here for more details).

citations <- dimensions_direct_citation %>% 
  add_count(publication_id) %>% 
  select(publication_id, n) %>% 
  unique

references_filtered <- dimensions_references %>% 
  left_join(citations) %>% 
  filter(n >= 5)

edges <- biblionetwork::biblio_cocitation(filter(dimensions_direct_citation, publication_id %in% references_filtered$publication_id), 
                                          "citing_id", 
                                          "publication_id",
                                          weight_threshold = 3)
edges

##                  from             to     weight         Source         Target
##     1: pub.1000001876 pub.1008725945 0.18814417 pub.1000001876 pub.1008725945
##     2: pub.1000001876 pub.1008947877 0.03468730 pub.1000001876 pub.1008947877
##     3: pub.1000001876 pub.1013813490 0.07749843 pub.1000001876 pub.1013813490
##     4: pub.1000001876 pub.1050095878 0.14301939 pub.1000001876 pub.1050095878
##     5: pub.1000001876 pub.1064524465 0.06073322 pub.1000001876 pub.1064524465
##    ---                                                                       
## 71877: pub.1119779752 pub.1125952654 0.40824829 pub.1119779752 pub.1125952654
## 71878: pub.1134024220 pub.1135197325 0.10846523 pub.1134024220 pub.1135197325
## 71879: pub.1134024220 pub.1135197780 0.16888013 pub.1134024220 pub.1135197780
## 71880: pub.1142606957 pub.1142611529 0.13155870 pub.1142606957 pub.1142611529
## 71881: pub.1142606957 pub.1143248862 0.10241831 pub.1142606957 pub.1143248862

We can now create a graph using tidygraph (Pedersen 2020) and networkflow.⁸

references_filtered <- references_filtered %>% 
  relocate(publication_id, .before = authors) # first column has to be the identifier

graph <- tbl_main_component(nodes = references_filtered, 
                            edges = edges, 
                            directed = FALSE)
graph

## # A tbl_graph: 3561 nodes and 71876 edges
## #
## # An undirected simple graph with 1 component
## #
## # Node Data: 3,561 x 12 (active)
##   Id    authors author_id source year  volume issue pagination doi   times_cited
##   <chr> <chr>   <chr>     <chr>  <chr> <chr>  <chr> <chr>      <chr> <chr>      
## 1 pub.~ [Barro~ []        The Q~ 2006  121    "3"   823-866    10.1~ 1133       
## 2 pub.~ [Gouri~ []        Ameri~ 2012  102    "6"   2734-2766  10.1~ 372        
## 3 pub.~ [Isoré~ [ur.0161~ Journ~ 2017  79     ""    97-125     10.1~ 27         
## 4 pub.~ [Calvo~ [ur.0744~ Journ~ 1983  12     "3"   383-398    10.1~ 4413       
## 5 pub.~ [Chris~ [ur.0114~ Handb~ 2010  3      ""    285-367    10.1~ 126        
## 6 pub.~ [Heute~ [ur.0161~ Revie~ 2012  15     "2"   244-264    10.1~ 161        
## # ... with 3,555 more rows, and 2 more variables: references <chr>, n <int>
## #
## # Edge Data: 71,876 x 5
##    from    to weight Source         Target        
##   <int> <int>  <dbl> <chr>          <chr>         
## 1    33   867 0.188  pub.1000001876 pub.1008725945
## 2     4   867 0.0347 pub.1000001876 pub.1008947877
## 3    12   867 0.0775 pub.1000001876 pub.1013813490
## # ... with 71,873 more rows

set.seed(1234)
graph <- leiden_workflow(graph) # identifying clusters of nodes 

nb_communities <- graph %>% 
  activate(nodes) %>% 
  as_tibble %>% 
  select(Com_ID) %>% 
  unique %>% 
  nrow
palette <- scico::scico(n = nb_communities, palette = "hawaii") %>% # creating a color palette
    sample()
  
graph <- community_colors(graph, palette, community_column = "Com_ID")

graph <- graph %>% 
  activate(nodes) %>%
  mutate(size = n,# will be used for size of nodes
         first_author = str_extract(authors, "(?<=\\[)(.+?)(?=,)"),
         label = paste0(first_author, "-", year)) 

graph <- community_names(graph, 
                         ordering_column = "size", 
                         naming = "label", 
                         community_column = "Com_ID")

graph <- vite::complete_forceatlas2(graph, 
                                    first.iter = 10000)


top_nodes  <- top_nodes(graph, 
                        ordering_column = "size", 
                        top_n = 15, 
                        top_n_per_com = 2,
                        biggest_community = TRUE,
                        community_threshold = 0.02)
community_labels <- community_labels(graph, 
                                     community_name_column = "Community_name",
                                     community_size_column = "Size_com",
                                     biggest_community = TRUE,
                                     community_threshold = 0.02)

A co-citation network allows us to observe what are the main influences of a field of research. At the centre of the network, we find the most cited references. On the borders of the graph, there are specific communities that influence different parts of the literature on DSGE. Here, the size of nodes depends on the number of times they are cited in our corpus.

If we compare to the Scopus co-citation network, we find some similarities as the domination of the Smets and Wouters (2007) community.⁹ However, the network is less dense here and displays more marginal communities. The “NA-1994” community is a good example of what I was talking about above on Dimensions identifying process. “NA-1994” (the biggest node of this community) is actually Hamilton (1994).

graph <- graph %>% 
  activate(edges) %>% 
  filter(weight > 0.05)

ggraph(graph, "manual", x = x, y = y) + 
  geom_edge_arc0(aes(color = color_edges, width = weight), alpha = 0.4, strength = 0.2, show.legend = FALSE) +
  scale_edge_width_continuous(range = c(0.1,2)) +
  scale_edge_colour_identity() +
  geom_node_point(aes(x=x, y=y, size = size, fill = color), pch = 21, alpha = 0.7, show.legend = FALSE) +
  scale_size_continuous(range = c(0.3,16)) +
  scale_fill_identity() +
  ggnewscale::new_scale("size") +
  ggrepel::geom_text_repel(data = top_nodes, aes(x=x, y=y, label = Label), size = 2, fontface="bold", alpha = 1, point.padding=NA, show.legend = FALSE) +
  ggrepel::geom_label_repel(data = community_labels, aes(x=x, y=y, label = Community_name, fill = color), size = 6, fontface="bold", alpha = 0.9, point.padding=NA, show.legend = FALSE) +
  scale_size_continuous(range = c(0.5,5)) +
  theme_void()

Figure 5: Bibliographic coupling network of articles using DSGE models

The differences between Scopus and Dimensions co-citation network would deserve a more thorough investigation, which is outside the scope of this tutorial.¹⁰ So let’s conclude just by observing what are the most cited nodes in each community. Here, we can see more clearly that community 02 with represent to a certain extent the “core” of the literature at the basis of DSGE models. Community 04 deals more with econometric problems, while community 05 tackles international issues.

ragg::agg_png(here("content", "en", "post", "2022-02-03-extracting-biblio-data-2", "top_ref.png"),
              width = 30, 
              height = 25,
              units = "cm",
              res = 200)
top_nodes(graph,
          ordering_column = "size",
          top_n = 10,
          top_n_per_com = 6,
          biggest_community = TRUE,
          community_threshold = 0.03) %>% 
  select(Community_name, references, n, color) %>% 
  mutate(label = str_wrap(references, 35),
         label = tidytext::reorder_within(label, n, Community_name)) %>% 
  ggplot(aes(n, label, fill = color)) +
  geom_col(show.legend = FALSE) +
  scale_fill_identity() +
  facet_wrap(~Community_name, ncol = 3, scales = "free") +
  tidytext::scale_y_reordered() +
  labs(x = "Number of citations", y = NULL) +
  theme_classic(base_size = 10)
invisible(dev.off())

Figure 6: Most cited references per communities

References

Extracting data from Scopus

Using Scopus website

I focus here on a not so much historical subject because a very recent one: the Dynamic Stochastic General Equilibrium (DSGE) models. This type of models has emerged in the late 1990s and has become standard in academic publications as well as in policymaking institutions, notably in central banks.²

If you don’t have one, you will need to create an account on Scopus using your institutional email address. Once you are on the “Documents” query page, you can search for “DSGE” or “Dynamic Stochastic General Equilibrium” in documents titles, abstracts and keywords (see Figure 1). On January 2022, I got 2633 results. You can select “All” the documents and then click on the down arrow on the righ of “CSV export” (see Figure 2). You then have to choose all the information you want to download in a .csv. What we need is to cross the “Include References” box as we need it for the bibliographic co-citation later (see Figure 3).³

Figure 1: Scopus search page

Figure 2: Scopus results page

Figure 3: Choosing the information to export in a CSV

The .csv you have exported gathers metadata on authors, title, journal, abstract, keywords, affiliations and references on documents (mainly articles, but also conference papers, book chapters, and reviews) mentioning DSGE in their title, abstract or keywords. There are two limits at this method for extracting data:

• the quantity of data you can export is limited to 2000, which is not much;
• part of the metadata are relatively raw, like affiliations and references, what involves some cleaning.

I will show you how to clean these raw data in the next section. But the following sub-section explains you how to use Scopus APIs directly in R, to query more easily for different data and extract more items.

api_key <- "your_api_key"
set_api_key(api_key)

insttoken <- "your_institutional_token"
insttoken <- inst_token_header(insttoken)

dsge_query <- rscopus::scopus_search("TITLE-ABS-KEY(DSGE) OR TITLE-ABS-KEY(\"Dynamic Stochastic General Equilibrium\")", 
                                     view = "COMPLETE",
                                     headers = insttoken)
dsge_data_raw <- gen_entries_to_df(dsge_query$entries)

dsge_papers <- dsge_data_raw$df
dsge_affiliations <- dsge_data_raw$affiliation 
dsge_authors <- dsge_data_raw$author

citing_articles <- dsge_papers$`dc:identifier` # extracting the IDs of our articles on dsge
citation_list <- list()

for(i in 1:length(citing_articles)){
  citations_query <- abstract_retrieval(citing_articles[i],
                                        identifier = "scopus_id",
                                        view = "REF",
                                        headers = insttoken)
  if(!is.null(citations_query$content$`abstracts-retrieval-response`)){ # Checking if the article has some references before collecting them
    
    nb_ref <- as.numeric(citations_query$content$`abstracts-retrieval-response`$references$`@total-references`)
    citations <- gen_entries_to_df(citations_query$content$`abstracts-retrieval-response`$references$reference)$df
    
    if(nb_ref > 40){ # The loop to collect all the references
      nb_query_left_to_do <- floor((nb_ref) / 40)
      cat("Number of requests left to do :", nb_query_left_to_do, "\n")
      for (j in 1:nb_query_left_to_do){
        cat("Request n°", j , "\n")
        citations_query <- abstract_retrieval(citing_articles[i],
                                              identifier = "scopus_id",
                                              view = "REF",
                                              startref = 40*j+1,
                                              headers = insttoken)
        citations_sup <- gen_entries_to_df(citations_query$content$`abstracts-retrieval-response`$references$reference)$df
        citations <- bind_rows(citations, citations_sup)
      }
    }
    
    citations <- citations %>% 
      as_tibble(.name_repair = "unique") %>%
      select_if(~!all(is.na(.)))
    
    citation_list[[citing_articles[i]]] <- citations
  }
}

dsge_references <- bind_rows(citation_list, .id = "citing_art")

Cleaning Scopus data

So let’s come back to the data we have downloaded on Scopus website:

#' # Cleaning scopus data from website search
#' 

scopus_search_1 <- read_csv(here(scopus_path, "scopus_search_1998-2013.csv"))
scopus_search_2 <- read_csv(here(scopus_path, "scopus_search_2014-2021.csv"))
scopus_search <- rbind(scopus_search_1, scopus_search_2) %>% 
  mutate(citing_id = paste0("A", 1:n())) %>% # We create a unique ID for each doc of our corpus
  clean_names() # janitor function to clean column names

scopus_search

## # A tibble: 2,608 × 23
##    authors      title  year source_title volume issue art_no page_start page_end
##    <chr>        <chr> <dbl> <chr>        <chr>  <chr> <chr>  <chr>      <chr>   
##  1 Ha J., So I. Infl…  2013 Global Econ… 42     4     <NA>   396        424     
##  2 Botero J., … Exog…  2013 Revista de … 16     1     <NA>   1          24      
##  3 Kirsanova T… Comm…  2013 Internation… 9      4     <NA>   99         151     
##  4 Ashimov A.A… Para…  2013 Economic De… <NA>   <NA>  <NA>   95         188     
##  5 Khan A., Th… Cred…  2013 Journal of … 121    6     <NA>   1055       1107    
##  6 Jeřábek T.,… Pred…  2013 Acta Univer… 61     7     <NA>   2229       2238    
##  7 Hu X., Xu B… Infl…  2013 Journal of … 5      12    <NA>   636        641     
##  8 Cha H.       Taki…  2013 Internation… 7      2     <NA>   280        296     
##  9 da Silva M.… The …  2013 North Ameri… 26     <NA>  <NA>   266        281     
## 10 Sandri D., … Fina…  2013 Journal of … 45     SUPP… <NA>   59         86      
## # ℹ 2,598 more rows
## # ℹ 14 more variables: page_count <dbl>, cited_by <dbl>, doi <chr>,
## #   affiliations <chr>, authors_with_affiliations <chr>, abstract <chr>,
## #   author_keywords <chr>, index_keywords <chr>, references <chr>,
## #   correspondence_address <chr>, language_of_original_document <chr>,
## #   document_type <chr>, source <chr>, citing_id <chr>

There are several things to clean:

• We have several authors per document. For some analysis, for instance co-authorship networks, it is better to have an “author table” which associates each author to a list of papers;
• We have several affiliations per article as well as several authors_with_affiliations. It allows us to connect authors with their affiliations, but here again we need to separate it in as many lines as authors (if I am not mistaken, it seems there is only one affiliation per author in this set of data);
• references;
• Possibly to separate author_keywords and index_keywords if you want to use it.

In what follows, we clean scopus_search in order to produce 3 additional data.frames:

• one data.frame which associates each article to a list of authors and their corresponding affiliation (see below);
• one data.frame which associate each article to the list of references it cites (one article has as many lines as the number of cited references): this is a “direct citation” table (see here);
• a list of all the references cited, which implies to find which references are the same in the direct citation table (see this sub-section)

affiliations_raw <- scopus_search %>% 
  select(citing_id, authors, affiliations, authors_with_affiliations)

knitr::kable(head(affiliations_raw, n = 2))

scopus_affiliations <- affiliations_raw %>% 
  separate_rows(authors, sep = ", ") %>% 
  separate_rows(contains("with"), sep = "; ") %>% 
  mutate(authors_from_affiliation = str_extract(authors_with_affiliations, 
                                                "^(.+?)\\.(?=,)"),
         authors_from_affiliation = str_remove(authors_from_affiliation, ","),
         affiliations = str_remove(authors_with_affiliations, "^(.+?)\\., "),
         country = str_remove(affiliations, ".*, ")) %>% # Country is after the last comma
  filter(authors == authors_from_affiliation) %>% 
  select(citing_id, authors, affiliations, country)


knitr::kable(head(scopus_affiliations))

#' ## Extracting and cleaning references
references_extract <- scopus_search %>% 
  filter(! is.na(references)) %>% 
  select(citing_id, references) %>% 
  separate_rows(references, sep = "; ") %>% 
  mutate(id_ref = 1:n()) %>% 
  as_tibble

knitr::kable(head(references_extract))

extract_authors <- ".*[:upper:][:alpha:]+( Jr(.)?)?, ([A-Z]\\.[ -]?)?([A-Z]\\.[ -]?)?([A-Z]\\.)?[A-Z]\\."
extract_year_brackets <- "(?<=\\()\\d{4}(?=\\))"
extract_pages <- "(?<= (p)?p\\. )([A-Z])?\\d+(-([A-Z])?\\d+)?"
extract_volume_and_number <- "(?<=( |^)?)\\d+ \\(\\d+(-\\d+)?\\)"

cleaning_references <- references_extract %>% 
  mutate(authors = str_extract(references, paste0(extract_authors, "(?=, )")),
         remaining_ref = str_remove(references, paste0(extract_authors, ", ")), # cleaning from authors
         is_article = ! str_detect(remaining_ref, "^\\([:digit:]{4}"), 
         year = str_extract(references, extract_year_brackets) %>% as.integer)

cleaning_articles <- cleaning_references %>% 
  filter(is_article == TRUE) %>% 
  mutate(title = str_extract(remaining_ref, ".*(?=\\(\\d{4})"), # pre date extraction
         journal_to_clean = str_extract(remaining_ref, "(?<=\\d{4}\\)).*"), # post date extraction
         journal_to_clean = str_remove(journal_to_clean, "^,") %>% str_trim("both"), # cleaning a bit the journal info column
         pages = str_extract(journal_to_clean, extract_pages), # extracting pages
         volume_and_number = str_extract(journal_to_clean, extract_volume_and_number), # extracting standard volument and number: X (X)
         journal_to_clean = str_remove(journal_to_clean, " (p)?p\\. ([A-Z])?\\d+(-([A-Z])?\\d+)?"), # clean from extracted pages
         journal_to_clean = str_remove(journal_to_clean, "( |^)?\\d+ \\(\\d+(-\\d+)?\\)"), # clean from extracted volume and number
         volume_and_number = ifelse(is.na(volume_and_number), str_extract(journal_to_clean, "(?<= )([A-Z])?\\d+(-\\d+)?"), volume_and_number), # extract remaining numbers
         journal_to_clean = str_remove(journal_to_clean, " ([A-Z])?\\d+(-\\d+)?"), # clean from remaining numbers
         journal = str_remove_all(journal_to_clean, "^[:punct:]+( )?[:punct:]+( )?|(?<=,( )?)[:punct:]+( )?([:punct:])?|[:punct:]( )?[:punct:]+( )?$"), # extract journal info by removing inappropriate punctuations
         first_page = str_extract(pages, "\\d+"),
         volume = str_extract(volume_and_number, "\\d+"),
         issue = str_extract(volume_and_number, "(?<=\\()\\d+(?=\\))"),
         publisher = ifelse(is.na(first_page) & is.na(volume) & is.na(issue) & ! str_detect(journal, "(W|w)orking (P|p)?aper"), journal, NA),
         book_title = ifelse(str_detect(journal, " (E|e)d(s)?\\.| (E|e)dite(d|urs)? "), journal, NA), # Incollection article: Title of the book here
         book_title = str_extract(book_title, "[A-z ]+(?=,)"), # keeping only the title of the book
         publisher = ifelse(!is.na(book_title), NA, publisher), # if we have an incollection article, that's not a book, so no publisher
         journal = ifelse(!is.na(book_title) | ! is.na(publisher), NA, journal), # removing journal as what we have is a book
         publisher = ifelse(is.na(publisher) & str_detect(journal, "(W|w)orking (P|p)?aper"), journal, publisher), # adding working paper publisher information in publisher column
         journal = ifelse(str_detect(journal, "(W|w)orking (P|p)?aper"), "Working Paper", journal))

cleaned_articles <- cleaning_articles %>% 
  select(citing_id, id_ref, authors, year, title, journal, volume, issue, pages, first_page, book_title, publisher, references)

cleaning_non_articles <- cleaning_references %>% 
  filter(is_article == FALSE) %>% 
  mutate(remaining_ref = str_remove(remaining_ref, "\\(\\d{4}\\)(,)? "),
         title = str_extract(remaining_ref, ".*(?=, ,)"),
         pages = str_extract(remaining_ref, "(?<= (p)?p\\. )([A-Z])?\\d+(-([A-Z])?\\d+)?"), # extracting pages
         volume_and_number = str_extract(remaining_ref, "(?<=( |^)?)\\d+ \\(\\d+(-\\d+)?\\)"), # extracting standard volument and number: X (X)
         remaining_ref = str_remove(remaining_ref, " (p)?p\\. ([A-Z])?\\d+(-([A-Z])?\\d+)?"), # clean from extracted pages
         remaining_ref = str_remove_all(remaining_ref, ".*, ,"), # clean dates and already extracted titles
         remaining_ref = str_remove(remaining_ref, "( |^)?\\d+ \\(\\d+(-\\d+)?\\)"), # clean from extracted volume and number
         volume_and_number = ifelse(is.na(volume_and_number), str_extract(remaining_ref, "(?<= )([A-Z])?\\d+(-\\d+)?"), volume_and_number), # extract remaining numbers
         remaining_ref = str_remove(remaining_ref, " ([A-Z])?\\d+(-\\d+)?"), # clean from remaining numbers
         journal = ifelse(str_detect(remaining_ref, "(W|w)orking (P|p)aper"), "Working Paper", NA),
         journal = ifelse(str_detect(remaining_ref, "(M|m)anuscript"), "Manuscript", journal),
         journal = ifelse(str_detect(remaining_ref, "(M|m)imeo"), "Mimeo", journal),
         publisher = ifelse(is.na(journal), remaining_ref, NA) %>% str_trim("both"),
         first_page = str_extract(pages, "\\d+"),
         volume = str_extract(volume_and_number, "\\d+"),
         issue = str_extract(volume_and_number, "(?<=\\()\\d+(?=\\))"),
         book_title = NA) # to be symetric with "cleaned_articles"

cleaned_non_articles <- cleaning_non_articles %>% 
  select(citing_id, id_ref, authors, year, title, journal, volume, issue, pages, first_page, book_title, publisher, references)

# merging the two files.
cleaned_ref <- rbind(cleaned_articles, cleaned_non_articles)

#' Now we have all the references, we can do a bit of cleaning on the authors name,
#' and extract useful information, like DOI, for matching later.

cleaned_ref <- cleaned_ref %>% 
  mutate(authors = str_remove(authors, " Jr\\."), # standardising authors name to favour matching later
         authors = str_remove(authors, "^\\(\\d{4}\\)(\\.)?( )?"),
         authors = str_remove(authors, "^, "),
         authors = ifelse(is.na(authors), str_extract(references, ".*[:upper:]\\.(?= \\d{4})"), authors), # specific case
         journal = str_remove(journal, "[:punct:]$"), # remove unnecessary punctuations at the end
         doi = str_extract(references, "(?<=DOI(:)? ).*|(?<=\\/doi\\.org\\/).*"),
         pii = str_extract(doi, "(?<=PII ).*"),
         doi = str_remove(doi, ",.*"), # cleaning doi
         pii = str_remove(pii, ",.*"), # cleaning pii
  )

knitr::kable(head(cleaned_ref, n = 3))

cleaned_ref <- cleaned_ref %>%
  mutate(first_author = str_extract(authors, "^[[:alpha:]+[']?[ -]?]+, ([A-Z]\\.[ -]?)?([A-Z]\\.[ -]?)?([A-Z]\\.)?[A-Z]\\.(?=(,|$))"),
         first_author_surname = str_extract(first_author, ".*(?=,)"),
         across(.cols = c("authors", "first_author", "journal", "title"), ~toupper(.))) 

matching_ref <- function(data, id_ref, ..., col_name){
  match <- data %>% 
    group_by(...) %>% 
    mutate(new_id = min({{id_ref}})) %>% 
    drop_na(...) %>% 
    ungroup() %>% 
    select({{id_ref}}, new_id) %>% 
    rename_with(~ paste0(col_name, "_new_id"), .cols = new_id)
  
  data <- data %>% 
    left_join(match)
}

identifying_ref <- cleaned_ref %>%
  matching_ref(id_ref, first_author_surname, year, title, col_name = "fayt") %>% 
  matching_ref(id_ref, journal, volume, issue, first_page, col_name = "jvip") %>% 
  matching_ref(id_ref, authors, year, volume, first_page, col_name = "ayvp") %>% 
  matching_ref(id_ref, first_author_surname, year, volume, first_page, col_name = "fayvp") %>%
  matching_ref(id_ref, title, year, first_page, col_name = "typ") %>% 
  matching_ref(id_ref, pii, col_name = "pii") %>% 
  matching_ref(id_ref, doi, col_name = "doi") 

direct_citation <- identifying_ref %>%  
  mutate(new_id_ref = select(., ends_with("new_id")) %>%  reduce(pmin, na.rm = TRUE),
         new_id_ref = ifelse(is.na(new_id_ref), id_ref, new_id_ref))  %>% 
  relocate(new_id_ref, .after = citing_id) %>% 
  select(-id_ref & ! ends_with("new_id"))

knitr::kable(head(filter(direct_citation, new_id_ref == 2), n = 4))

important_info <- c("authors",
                    "year",
                    "title",
                    "journal",
                    "volume",
                    "issue",
                    "pages",
                    "book_title",
                    "publisher")
references <- direct_citation %>% 
  mutate(nb_na = rowSums(!is.na(select(., all_of(important_info))))) %>% 
  group_by(new_id_ref) %>% 
  slice_max(order_by = nb_na, n = 1, with_ties = FALSE) %>% 
  select(-citing_id) %>% 
  unique

knitr::kable(head(references, n = 4))

corpus <- scopus_search %>% 
select(-references)

ref_text <- paste0(references_extract$references, collapse = "\\\n")
name_file <- "ref_in_text.txt"
write_file(ref_text,
           here(scopus_path,
                name_file))

destination_anystyle <- "anystyle_cleaned"

directory_command <- paste0("cd ", scopus_path)
anystyle_command <- paste0("anystyle -f bib parse ",
                           name_file,
                           " ",
                           destination_anystyle)

options(encoding = "UTF-8")
bib_ref <- bib2df(here(scopus_path,
                       destination_anystyle,
                       "ref_in_text.bib"))
bib_ref <- bib_ref %>% 
  janitor::clean_names() %>% 
  select_if(~!all(is.na(.))) %>%  # removing all empty columns
  mutate(id_ref = 1:n()) %>% 
  select(-c(translator, citation_number, arxiv, director, source))

knitr::kable(head(bib_ref))

Exploring the DSGE literature

Now we have our bibliographic data, the first thing we can look at is the most cited references in our corpus.

direct_citation %>% 
  add_count(new_id_ref) %>% 
  select(new_id_ref, n) %>% 
  unique() %>% 
  slice_max(n, n = 10) %>%
  left_join(select(references, new_id_ref, references)) %>% 
  select(references, n) %>% 
  knitr::kable()

As we have the affiliations, we can try to see which are the top references for economists based in different countries:

direct_citation %>% 
  left_join(select(scopus_affiliations, citing_id, country)) %>% 
  unique() %>% 
  group_by(country) %>% 
  mutate(nb_article = n()) %>% 
  filter(nb_article > 5000) %>% # we keep only countries with 5000 articles
  add_count(new_id_ref) %>%
  select(new_id_ref, n) %>% 
  unique() %>% 
  slice_max(n, n = 8) %>%
  left_join(select(references, new_id_ref, references)) %>% 
  select(references, n)  %>% 
  mutate(label = str_extract(references, ".*\\(\\d{4}\\)") %>% 
           str_wrap(30),
         label = tidytext::reorder_within(label, n, country)) %>% 
  ggplot(aes(n, label, fill = country)) +
  geom_col(show.legend = FALSE) +
  facet_wrap(~country, ncol = 3, scales = "free") +
  tidytext::scale_y_reordered() +
  labs(x = "Number of citations", y = NULL) +
  theme_classic(base_size = 10)

Figure 4: Most cited references per countries

By using affiliations we can observe a regional preference pattern: in European countries, economists tend to cite more Smets and Wouters (2003) (the source of the European Central Bank DSGE model) than Christiano, Eichenbaum, and Evans (2005), while the opposite is true in the United States (Smets and Wouters 2007 is on US data). We can also notice that Kydland and Prescott (1982) is less popular in continental Europe.

Bibliographic co-citation analysis

To conclude this (long) tutorial, we can build a co-citation network: the references we have matched are the nodes of the network, and they are linked together depending on the number of times they are cited together (or in other words, the number of times they are together in a bibliography). We use the biblio_cocitation function of the biblionetwork package. The edge between two nodes is weighted depending of the total number of times each reference has been cited in the whole corpus (see here for more details).

citations <- direct_citation %>% 
  add_count(new_id_ref) %>% 
  select(new_id_ref, n) %>% 
  unique

references_filtered <- references %>% 
  left_join(citations) %>% 
  filter(n >= 5)

edges <- biblionetwork::biblio_cocitation(filter(direct_citation, new_id_ref %in% references_filtered$new_id_ref), 
                                          "citing_id", 
                                          "new_id_ref",
                                          weight_threshold = 3)
edges

## Key: <Source>
##          from     to    weight Source Target
##        <char> <char>     <num>  <int>  <int>
##     1:      2      4 0.2354379      2      4
##     2:      2      5 0.1244966      2      5
##     3:      2      6 0.0518193      2      6
##     4:      2      7 0.1623352      2      7
##     5:      2     14 0.1349191      2     14
##    ---                                      
## 42657:  64761  64841 0.4045199  64761  64841
## 42658:  64841  64936 0.3481553  64841  64936
## 42659:  66416  68931 0.7453560  66416  68931
## 42660:  66416  68935 0.5039526  66416  68935
## 42661:  68931  68935 0.6761234  68931  68935

We can then take our corpus and these edges to create a network/graph thanks to tidygraph (Pedersen 2020) and networkflow. I don’t enter in the details here as that is not the purpose of this tutorial and that the different steps are explained on networkflow website.

graph <- tbl_main_component(nodes = references_filtered, 
                            edges = edges, 
                            directed = FALSE)
graph

## # A tbl_graph: 2836 nodes and 42661 edges
## #
## # An undirected simple graph with 1 component
## #
## # A tibble: 2,836 × 18
##      Id authors      year title journal volume issue pages first_page book_title
##   <int> <chr>       <int> <chr> <chr>   <chr>  <chr> <chr> <chr>      <chr>     
## 1     2 CALVO, G.    1983 "STA… JOURNA… 12     3     383-… 383        <NA>      
## 2     4 DIXIT, A.,…  1977 "MON… AMERIC… 67     3     297-… 297        <NA>      
## 3     5 GERALI, A.…  2010 "CRE… WORKIN… 42     6     107-… 107        <NA>      
## 4     6 GOODFRIEND…  2007 "BAN… JOURNA… 54     5     1480… 1480       <NA>      
## 5     7 IACOVIELLO…  2005 "HOU… AMERIC… 95     3     739-… 739        <NA>      
## 6    14 ROTEMBERG,…  1982 "MON… REVIEW… 49     4     517-… 517        <NA>      
## # ℹ 2,830 more rows
## # ℹ 8 more variables: publisher <chr>, references <chr>, doi <chr>, pii <chr>,
## #   first_author <chr>, first_author_surname <chr>, nb_na <dbl>, n <int>
## #
## # A tibble: 42,661 × 5
##    from    to weight Source Target
##   <int> <int>  <dbl>  <int>  <int>
## 1     1     2 0.235       2      4
## 2     1     3 0.124       2      5
## 3     1     4 0.0518      2      6
## # ℹ 42,658 more rows

set.seed(1234)
graph <- leiden_workflow(graph) # identifying clusters of nodes 

nb_communities <- graph %>% 
  activate(nodes) %>% 
  as_tibble %>% 
  select(Com_ID) %>% 
  unique %>% 
  nrow
palette <- scico::scico(n = nb_communities, palette = "hawaii") %>% # creating a color palette
    sample()
  
graph <- community_colors(graph, palette, community_column = "Com_ID")

graph <- graph %>% 
  activate(nodes) %>%
  mutate(size = n,# will be used for size of nodes
         label = paste0(first_author_surname, "-", year)) 

graph <- community_names(graph, 
                         ordering_column = "size", 
                         naming = "label", 
                         community_column = "Com_ID")

graph <- vite::complete_forceatlas2(graph, 
                                    first.iter = 10000)


top_nodes  <- top_nodes(graph, 
                        ordering_column = "size", 
                        top_n = 15, 
                        top_n_per_com = 2,
                        biggest_community = TRUE,
                        community_threshold = 0.02)
community_labels <- community_labels(graph, 
                                     community_name_column = "Community_name",
                                     community_size_column = "Size_com",
                                     biggest_community = TRUE,
                                     community_threshold = 0.02)

A co-citation network allows us to observe what are the main influences of a field of research. At the center of the network, we find the most cited references. On the borders of the graph, there are specific communities that influence different parts of the literature on DSGE. Here, the size of nodes depends on the number of times they are cited in our corpus.

graph <- graph %>% 
  activate(edges) %>% 
  filter(weight > 0.05)

ggraph(graph, "manual", x = x, y = y) + 
  geom_edge_arc0(aes(color = color_edges, width = weight), alpha = 0.4, strength = 0.2, show.legend = FALSE) +
  scale_edge_width_continuous(range = c(0.1,2)) +
  scale_edge_colour_identity() +
  geom_node_point(aes(x=x, y=y, size = size, fill = color), pch = 21, alpha = 0.7, show.legend = FALSE) +
  scale_size_continuous(range = c(0.3,16)) +
  scale_fill_identity() +
  ggnewscale::new_scale("size") +
  ggrepel::geom_text_repel(data = top_nodes, aes(x=x, y=y, label = Label), size = 2, fontface="bold", alpha = 1, point.padding=NA, show.legend = FALSE) +
  ggrepel::geom_label_repel(data = community_labels, aes(x=x, y=y, label = Community_name, fill = color), size = 6, fontface="bold", alpha = 0.9, point.padding=NA, show.legend = FALSE) +
  scale_size_continuous(range = c(0.5,5)) +
  theme_void()

Figure 5: Bibliographic coupling network of articles using DSGE models

Let’s conclude by observing what are the most cited nodes in each community. We see that community 04 deals with international issues while the 07 is linked to fiscal policy issues.

ragg::agg_png(here("content", "en", "post", "2022-01-31-extracting-biblio-data-1", "top-ref-country-1.png"),
              width = 35, 
              height = 30,
              units = "cm",
              res = 200)
top_nodes(graph,
          ordering_column = "size", 
          top_n_per_com = 6,
          biggest_community = TRUE,
          community_threshold = 0.04) %>% 
  select(Community_name, Label, title, n, color) %>% 
  mutate(label = paste0(Label, "-", title) %>% 
           str_wrap(34),
         label = tidytext::reorder_within(label, n, Community_name)) %>% 
  ggplot(aes(n, label, fill = color)) +
  geom_col(show.legend = FALSE) +
  scale_fill_identity() +
  facet_wrap(~Community_name, ncol = 3, scales = "free") +
  tidytext::scale_y_reordered() +
  labs(x = "Number of citations", y = NULL) +
  theme_classic(base_size = 11)
invisible(dev.off())

Figure 6: Most cited references per communities

References

Why learning R?

Debates about the use of quantitative methods when studying the history of economics have flourished in the last years—see for instance the special issue in the Journal of Economic Methodology (Edwards, Giraud, and Schinckus 2018)—and many recent contributions have used bibliometrics, network analysis or topic-modelling.¹ When the temptation is growing in you to try your hand at quantitative methods (as it has been the case for me a couple of years ago), the first question is likely to be “but how can I do, and which tools should I learn to use?”

It exists many software (some free, others to buy) that would allow you to use bibliometrics, networks analysis or topic-modelling. Depending on what you want to do, you will have to learn each corresponding software—after being sure that the tool is appropriate to accomplish the task you have in mind. That is what I did when I wanted to use network analysis for one of my paper: I learned how to use the very good GEPHI. However, Gephi is great when you want to manipulate networks, but at the condition that you have already built your network. And if you want to use other types of quantitative analysis on your data (for instance text analysis and topic-modelling), you will need to find another software. That is why, after this first research project integrating quantitative methods, I took another direction: I learned to program in R.²

Obviously, learning R has a higher entry cost than learning GEPHI—2 or 3 hours should be sufficient to learn the basics of the latter. But it offers many advantages and, in the medium- and long-term, large economies of scale:

• It would allow you to manipulate more easily and more quickly your data than by using excel (either programmatically or manually). You will be able to do things you cannot do in excel. And this becomes more and more true the larger your dataset.

• You will be able to use different quantitative methods (bibliometrics, network analysis, natural language processing, econometrics, machine learning, etc.) and to build visualisations just by learning one unique language (thanks to the many packages implemented in R).

• Writing “scripts” in R allows you to re-run the same tasks without any supplementary effort each time your data are changing or you want to change a parameter of your analysis. Besides, these scripts are a resource that you will be able to re-use in other projects.

• Outside of quantitative methods strictly speaking, learning R will enable you to do several other things:

  • writing books, articles, reports and slides efficiently thanks to Rmarkdown (Allaire et al. 2021);
  • publishing online your analyses and building dashboards and applications.³

If you are convinced that programming in R could be useful for you as a historian of economics, you now need some resources to learn to code. Here is a list of useful packages and tutorials I wish I had when I started learning to program in R and which I have used extensively since then to improve my skills.

First step: the basics

The very first step is obviously to download and install R and Rstudio on your computer, and to understand the basics of R language and how to use Rstudio. The first chapter of McConville (2021) or this lesson from R-ladies Sydney are good places to start from.

Once you are a bit familiar with what R is and the functioning of Rstudio, I think the priority is then to become familiar with the tydiverse collection of packages (Wickham 2021a; Wickham et al. 2019). A very reliable guide here is obviously Wickham and Gromelund’s (2017) R for Data Science. In this book, you will learn how to import data, how to transform them, how to visualise them and how to apply statistical models to them.⁴.

Another useful detour when you have progressively learned the basics of R, is to understand how to organise your workflow with R and Rstudio (how to organise your files? Why and how to create an “RStudio project”?): you can find useful information in the second chapter of Jenny Bryan (2017). I don’t want to overload the boat too soon, but when you will have begun your first project, it could be useful to learn the basis of version control and git. That is something that will be indispensable when you will work collectively on a project in Rstudio. Because using git is not very intuitive and because you will probably tear your hair out at first, it is better to become accustomed to use git from the beginning even (and above all?) when you are working alone. You will be confronted to simple cases and you will be less afraid to use git versioning in collective projects then. Jennyfer Bryan (2019) offers a rich documentation to learn to use git and github—the latter being fundamental to share your code and give explanations on the projects you are working on.⁵

R offers great tools for building a large variety of visualisations. The package ggplot2 (Wickham et al. 2021; Wickham 2016) is obviously the first of these tools, and you can find a dense documentation here. Kieran Healy’s (2018) book is also a good read to learn ggplot2 while thinking about the best practices to build data visualisations. One of the advantages of the ggplot2 is that it is accompanied by many extensions (new types of visualisations, tools to customize in different ways the visualisations produced with ggplot2, new themes and colour palettes, animation, etc) listed here. I acknowledge that learning this “grammar of graphics” is far from easy at the beginning, for you will have to know many functions and take into account many features when producing quantitative visualisations. The esquisse extension (Meyer and Perrier 2021) could allow you to reduce the cost of entry and to get used to the functions of ggplot2 gradually: it is an Rstudio add-in that allows you to build interactively your visualisations.

If you want to produce dynamic and interactive visualisations, the package plotly (Sievert et al. 2021; Sievert 2020) offers many possibility. The advantage of plotly is that, for some basic features, you can convert the visualisation produced with ggplot2 to interactive visualisations with just one line of code. But plotly for R also allows more complex customisation that necessitate to investigate more in-depth the different functions of the package (see a detailed documentation here).

This is it for the basics: obviously, you won’t master all the packages above in one day, and you will progressively get the hang of them in working on your own research project. However, you will likely need most of them in all the quantitative projects you will start, whether you want to do some simple statistical analysis on a prosopographic database of economists trained at the MIT, a comparison of the words used by Adam Smith and David Ricardo, or a bibliometric analysis of the publications about discrimination in economic academic journals. The following tools I will present you target more specific uses.

Learning bibliometrics and network analysis

Bibliometrics is useful for historians of economics to understand how the discipline has evolved, what are the major influences for specific sub-fields, how different communities have structured the discipline at different periods, etc.⁶ Network analysis comes as a complement of bibliometrics to produce bibliographic coupling, co-citation, keyword or co-authorship networks.

A first package, bibliometrix, offers different functions from extracting bibliometric data from different platforms (like Web of Science or Scopus), to computing statistics and building networks. I have never used it extensively, for two reasons:

1. I was working on large sets of data and bibliometrix appeared a bit slow for that;
2. I found the functions a bit like black boxes, and not really compatible with the tidyverse spirit. However, it seems very rich and I am sure it worth exploring.

Once you have your data—which is far from being an easy step⁷—I have developed, with the help of Alexandre Truc and François Claveau, a package to make the creation of very large networks of bibliographic data easy and fast: biblionetwork (Goutsmedt, Truc, and Claveau 2023). What you need as an input is a corpus and the references cited by this corpus, and you will be able to create easily a list of links between articles, references, authors, institutions, etc., with different measures of weights for these links.

Once you have your edges/links, you can create a network with the help of the igraph package (Csardi and Nepusz 2006). igraph is a very rich package with many functions to calculate network statistics (like centrality measures), find clusters and arrange the nodes in space. However, manipulating networks with igraph is not always very intuitive. Fortunately, the tidygraph package (Pedersen 2020) allows you to transform the networks in “tidy” data, in the tidyverse spirit, and to manipulate the networks accordingly. Similarly, the ggraph package (Pedersen 2021), which is an extension of ggplot2, allows the production of nice network visualisations.

Figure 1: Network example from the Mapping Macroeconomics project

Alexandre Truc and I are currently developing another package, networkflow, to facilitate the manipulation of networks, thanks to tidygraph and ggraph. The goal is to build some routines to mark out and accelerate the analysis of networks, from network creation, through clustering and spatialisation, to visualisation. In a close future, the package will integrate some tools for dynamic network analysis (by comparing the evolution of different networks over time).

Try your hand at text mining and topic modelling

Text mining is an essential approach to analyse large corpus and thus can bring useful information for historians of economics.

The first step is to transform the text of your documents in readable data in R. It first implies that the text of your documents is recognisable. To check that, you can just copy the text of a random page of your corpus and paste it on a text editor. If the text is not recognisable, or if the quality is not good, you should run Optical Character Recognition. In my former projects, I have used a very efficient OCR via a purchased software, PDF Element. I found it better than the tesseract OCR engine. However, I did not push the possibilities of the engine to the end. tesseract has been implemented in R in a package (Ooms 2021b). If you have pdf and that the text recognition is good, you can use the pdftools package (Ooms 2021a) to extract the text and manipulate it with R.

Once you have the text of your corpus ready to be imported in R, you can begin the analysis. Julia Silge and David Robinson (2020) have written a great introductory book to text mining in R. It relies on the tidytext package (Robinson and Silge 2021; Silge and Robinson 2016), which brings simple functions to do complicated things (tokenizing texts, calculating statistics like TF-IDF, etc.). If you want to know more about tokenization, stemming and lemmatization, stop words, or words embeddings, you can move to the next step and read the first part of Supervised Machine Learning for Text Analysis in R (Silge and Hvitfeldt 2021).

Once you have acquired some mastery for extracting text, cleaning and preprocessing it, and calculating standard statistics, you can be interested by more complex techniques and machine learning. Here, topic modelling can offer very rich results. It refers to the use of a model (the Latent Dirichlet Allocation being the most used) to identify k topics (k being determined by the modeller) in your corpus. The result of your topic modelling is that you will have each document of your corpus represented as a mixture of topics, and each topic represented as a mixture of words. To run topic modelling in R, you will find interesting information in Silge and Robinson’s (2020) corresponding chapter. In this chapter, they are using the topicmodels package (Grün and Hornik 2021, 2011).⁸ But I find that the stm package (Roberts, Stewart, and Tingley 2020) is also very useful with very nice functions implemented in it. It allows you to cross your topic modelling with the meta data of your corpus.

Finally, to help you running topic modelling in R (but also for other text mining issues), you can check the tips and good practices discussed by Welbers, Van Atteveldt, and Benoit (2017) or Banks et al. (2018). More generally, R offers many resources for text mining and Natural Language Processing (NLP): you can find an overview of the different existing packages here.

Figure 2: Natural Language Processing ecosystem in R

Digging into modelling and machine learning

I am not a specialist of modelling and machine learning at all, but I hope I will have the occasion to train myself in the next years. Indeed, there are many occasions it can be useful for historians. If you want to refresh your memory in econometrics why learning R, you can look at Schmelzer’s (2021) book that you can read online. Regarding machine learning, which can have numerous applications in history of economics, notably with classification techniques, I found that Greenwell (2020) offers a nice introduction to the implementation in R of different machine learning algorithms—you can read it here.

To help R users in their modelling job, the tidymodels collection of packages can be very useful, whether for preprocessing your data, evaluating your model or tuning your parameters. The website has also many interesting tutorials to learn the tidymodels approach.

Programming further: Functions, functions, functions… and packages

Once you move forward in mastering these different methods and improve your skills in R coding, an important task is to solidify your code and make it reliable for future work. The first step when writing code, is to document it: give your script a clear structure, explain why your are using this function, keep a trace of failed paths, point out what you could improve, etc. This could be useful for collaborators or readers of your scripts, but above all it will be useful for future you, as a messy script is the best way to lose a lot of time two or three months later when you will have to come back to your project (for instance after having received referees’ reports on your paper). You can find some useful advice here about structuring your script and naming objects. As Wickham and Grolemund (2017) explain here, it is also quite useful to come back to your script and to rewrite it to make it simpler and clearer.

An important step in this rewriting is to transform some of your code in functions. A good general rule is that if your are copying and pasting the same part of your code more than twice, it could be useful to transform it into a function. You will find useful content about how to program functions properly in Wickham (2019), beginning by the chapter on functions.⁹

If you have a set of functions that you are using regularly in one or several of your projects, you should consider writing a package. Even if you do not plan initially to share your code, writing a package is a good discipline for writing good code and for reproducibility. Later on, it would be useful for circulating your code and, perhaps, if you are satisfied with the result and think it could be useful to other R coders, you will want to publish it on CRAN. For writing a package, Wickham and Bryan (2021) offers an essential guide that you can also find online here. If you want to share your package, the pkgdown helps you to build easily a website documenting your package.

Writing and disseminating your work with R Mardown

Finally, you will probably need to circulate and publish your results. Fortunately, R Markdown is the perfect tool to do it. Outside of being able to run code within your R markdown document and thus to display the results, it offers many advantages like bibliographic management and Zotero integration.¹⁰ It allows you to build html, as well as pdf and word documents. For pdf, it relies on LaTeX and uses Pandoc for conversion from the markdown language.¹¹ With R Markdown, you will be able to write (automatised) reports and dashboards, academic articles, books and thesis dissertation (Xie 2021, 2016), slides, and even build you own website (Xie, Dervieux, and Presmanes Hill 2021; Xie, Hill, and Thomas 2017) and applications (Chang et al. 2021; Wickham 2021b).

To learn to use R Markdown, you can refer to Xie, Allaire, and Grolemund (2018) and Riederer, Xie, and Dervieux (2021) with the corresponding website here and here.

References

Usage

The basic function of the package is the biblio_coupling() function. This function calculates the number of references that different articles share together, as well as the coupling angle value of edges in a bibliographic coupling network (Sen and Gan 1983). What you need is just a file with entities (documents, authors, universities, etc.) citing references.¹

This example use the data incorporated in the package.²

library(biblionetwork)
biblio_coupling(Ref_stagflation, 
                source = "Citing_ItemID_Ref", 
                ref = "ItemID_Ref", 
                normalized_weight_only = FALSE, 
                weight_threshold = 1)

##             from         to     weight nb_shared_references     Source
##    1:     214927    2207578 0.14605935                    4     214927
##    2:     214927    5982867 0.04082483                    1     214927
##    3:     214927    8456979 0.09733285                    3     214927
##    4:     214927   10729971 0.29848100                    7     214927
##    5:     214927   16008556 0.04714045                    1     214927
##   ---                                                                 
## 2712: 1111111161 1111111172 0.03434014                    1 1111111161
## 2713: 1111111161 1111111180 0.02003610                    1 1111111161
## 2714: 1111111161 1111111183 0.04050542                    2 1111111161
## 2715: 1111111172 1111111180 0.03646625                    1 1111111172
## 2716: 1111111182 1111111183 0.27060404                    8 1111111182
##           Target
##    1:    2207578
##    2:    5982867
##    3:    8456979
##    4:   10729971
##    5:   16008556
##   ---           
## 2712: 1111111172
## 2713: 1111111180
## 2714: 1111111183
## 2715: 1111111180
## 2716: 1111111183

What you can do with biblionetwork

The package incorporates different types of method for calculating edges weight, depending on the type of network your working on and the variables you want to take into account. You will find on the package website a detailed vignette on these different methods and how to use the package for different networks.

Acknowledgements

The original function to find edges and calculate weights for large networks was developed by François Claveau. The different functions in this package have been developed, from Claveau’s original idea, by Alexandre Truc and Aurélien Goutsmedt. The package is maintained by Aurélien Goutsmedt.

References

Introduction

Bibliometric networks are an interesting tool to study the evolution of a disciplinary field or of a topic. A bibliographic coupling network, by looking at the references shared by a corpus of documents, allows us to understand the intellectual structure of a field at a certain point of time. A bibliographic co-citation network, by looking at the number of times the references cited by a corpus are cited together, reveals the main influences of a field.¹

Figure 1: Bibliographic coupling and co-citation

That is the two main tools I have used in my article “From the Stagflation to the Great Inflation: explaining the US economy of the 1970s” (Goutsmedt 2021). I analysed the way economists explained the US stagflation of the 1970s (a period of high unemployment and inflation) over time. Bibliometric networks completed a qualitative reading of the relevant articles and books and helped me to underline how we went from an explanation focusing on the effects of external shocks (the end of Bretton Woods and commodity-price shocks) on growth and unemployment, in the late 1970s and early 1980s, to an explanation that focused on the Federal Reserve mistakes and the effects of monetary policy on inflation.²

However, publishing academic articles with such tools often comes with some frustration. If bibliometric networks are helpful to prove your point and to display some of your results, they are also (and I am tempted to say “above all”) an “exploratory” tool that comes upstream of a project, well before having a clear idea of what you want to demonstrate.³ You can spend some time exploring your network, studying which nodes are in which communities, studying the effects of changing some parameters (in your partitioning algorithm for instance). But at the end of the day, what you will have in your paper is this:

Figure 2: My co-citation network for the 1997-2013 sub-period

Even if visually informative, the picture is an attempt of summarising all the information obtained by studying the network. And of course, it heavily relies on the explanations and interpretations given by its author, as it remains very hard for an external observer to understand what it is all about. Moreover, this observer is likely to experience frustration too, as she is unable to explore the network herself and depends on the visual choices made by the author.

The rationale behind this post is to evacuate a part of this frustration. I wanted to do several things here:

1. unveiling the database used for my article (and hoping that some people would like to use it in turn and complete/improve/expand it);
2. exposing some useful bits of R code to produce such networks. I am notably using two R packages that I am developing with Alexandre Truc: biblionetwork and networkflow. The second one heavily relies on igraph (Csardi and Nepusz 2006), tidygraph (Pedersen 2020b) and ggraph (Pedersen 2020a) great packages.
3. displaying interactive coupling and co-citation networks (the very reason for the long monologue above…).⁴

Before to go further, let’s charge some packages that will be useful for what follows:

cran_list <- c("data.table","magrittr","igraph","tidygraph","ggraph", "leidenAlg",
               "scico","DT","ggiraph","ggnewscale","ggrepel","htmlwidgets",
               "htmltools","widgetframe","sigmajs","dplyr","devtools")
for(p in cran_list){
  if (p %in% installed.packages()==FALSE){install.packages(p,dependencies = TRUE)}
  library(p,character.only=TRUE)
}

github_list <-  c("ParkerICI/vite","agoutsmedt/biblionetwork","agoutsmedt/networkflow")
for(p in github_list){
  if (gsub(".*/","",p) %in% installed.packages()==FALSE){devtools::install_github(p)}
  library(gsub(".*/","",p),character.only=TRUE)
}

Building a database of stagflation explanations

The first step in my article on stagflation was to decide what could be regarded as an explanation of the US stagflation and to identify the academic articles, books and book chapters responding positively to the criteria (see Goutsmedt 2021 for the whole list of criteria). I decided to focus on documents that devoted sufficient space to deal explicitly with the explanation of the phenomena.

To identify the corresponding documents, I relied on two methods:

• I took as a point of departure the 2008 NBER conference on “The Great Inflation: The Rebirth of Modern Central Banking”—published in 2013—and checked the bibliography of the conference articles dealing with the US economic situation to find new references about stagflation. I repeated the process with these new references and so on.
• I searched for all the articles with the term “Stagflation” or “Great Inflation” in their title on Web of Science (WoS) and integrated to my dataset all the articles dealing with the 1970s US stagflation. I also checked the bibliography of these documents and find new ones.

I ended up with 174 documents (articles in academic journals, working papers, books and book chapters). 104 were in the WoS database. I thus could extract the references cited by these articles (yes, that’s almost only academic articles that you find in WoS). The process for the 70 remaining documents was more time-consuming. I parsed the bibliography of these documents in anystyle and cleaned the results. Then, I had to match these new references with what I got first with WoS. I eventually ended up with 572 references cited at least twice by the 174 documents on the US stagflation. 91 stagflation documents were also part of the 572 references—meaning that 83 articles and books on stagflation were either not cited or only once by other articles or books on stagflation.

Here is all the stagflation documents and all the references cited at least twice by these documents:⁵  

Nodes_stagflation <- as.data.table(Nodes_stagflation)
data <- datatable(Nodes_stagflation[order(Type, decreasing = TRUE),c("Author","Year","Title","Journal","Type")], 
                  rownames = FALSE, 
                  options =  list(scrollY = "400px",
                                  scrollCollapse = TRUE,
                                  scrollX = "400px",
                                  paging = FALSE))

frameWidget(data)

If we look at the number of documents on stagflation per year, we see that there were two waves of interest for this topic:

Nodes_stagflation %>%
          filter(Type == "Stagflation") %>% 
          count(Year) %>%
        ggplot(aes(x = Year, y = n, fill = n)) +
        geom_bar(stat = "sum", show.legend = FALSE, ) +
        scale_fill_viridis(option = "A") +
        ylab("Number of Documents on Stagflation per Year") + 
        xlab("Year") +
        theme_minimal() +
        scale_x_discrete(breaks = seq(1975,2013,5))

Figure 3: Number of stagflation documents per year

I have thus decided to focus on these two waves and to study two subperiods: 1975-1986; 1997-2013.

Bibliometric coupling and co-citation analyses

For each sub-period, I build a coupling and a co-citation network. The biblionetwork package, with the biblio_coupling() function, allows us to build the edges of the network, from a direct citation data frame (what each stagflation document cites), and to calculate a weight for these edges. The method for calculating the weights takes into account the size of the bibliography of each document. The biblio_cocitation() function does the same for the co-citation edges and the weights measure takes into account the total number of citations of each reference.⁶

The networkflow package offers functions wrapped around other packages, to simplify the analysis of networks and their projections. For creating and manipulating the network, I use the tidygraph package, which wraps the functionnalities of the classic igraph package in a tidy format. I use the Leiden algorithm (Traag, Waltman, and van Eck 2019) to find communities—groups of nodes highly connected together—in the four networks. For structuring the network in a two-dimensional space, I use the Force Atlas 2 (FA2) layout (Jacomy et al. 2014). FA2 is a force-based layout: basically, connected nodes are attracted together, and the layout minimises edges crossing. For community colors, I use the scico package (Pedersen and Crameri 2020) which brings palettes taking into account colour blindness issues.

In what follows, I use a loop with conditions, to build the four networks and prepare their projection:

graph_list <- list() # empty list to fill in the loop

# fixing some variables for the loop
start_year <- c(1975,1997) 
end_year <- c(1986,2013)
network <- c("coupling","cocitation")
palette <- c("lapaz","roma")

for(i in 1:2){
  for(j in 1:length(network)){
    
# direct citation data, depending on the sub-period
ref <- as.data.table(Ref_stagflation) %>%
  .[Citing_ItemID_Ref %in% (as.data.table(Nodes_stagflation) %>% .[between(Year, start_year[i], end_year[i])])$ItemID_Ref]

# building the list of nodes, depending on whether we are building the coupling or co-citation network 
    if(network[j] == "coupling"){
  nodes <- as.data.table(Nodes_stagflation) 
    %>% .[between(Year, start_year[i], end_year[i]) & Type == "Stagflation"]     
    } else {
  nodes <- as.data.table(Nodes_stagflation) 
    %>% .[ItemID_Ref %in% ref$ItemID_Ref]     
    }

# calculating the number of citations of each node within the period
nb_cit <- ref[, .N, ItemID_Ref]
colnames(nb_cit)[colnames(nb_cit) == "N"] <- "size"
nodes <- merge(nodes, nb_cit, by.x = "ItemID_Ref", by.y = "ItemID_Ref", all.x = TRUE)

# setting the value of article with no citation to 0,5 to display them in the visualisation, 
#but at a smaller size than articles with one citation
nodes[is.na(size), size:= 0.5]

# finding the edges and calculating their weight using the `biblionetwork` package
 if(network[j] == "coupling"){
  edges <- biblio_coupling(ref, "Citing_ItemID_Ref", "ItemID_Ref")    
    } else {
  ref <- ref[ItemID_Ref %in% nodes$ItemID_Ref]     
  edges <- biblio_cocitation(ref, "Citing_ItemID_Ref", "ItemID_Ref")
    }

# creating the graph with `networkflow` package
graph <- tbl_main_component(edges = edges, 
                            nodes = nodes, 
                            directed = FALSE, 
                            node_key = "ItemID_Ref")

# finding communities 
graph <- leiden_workflow(graph, niter = 1000)

# giving colors

graph <- community_colors(graph, 
                          palette = c(scico(6, begin = 0.2, end = 0.9, palette = palette[j])))

# calculating coordinates using the package `vite` and its Force Atlas 2 implementation
graph <- vite::complete_forceatlas2(graph, 
                                    first.iter = 5000, 
                                    overlap.method = "repel", 
                                    overlap.iter = 1000)

graph_list[[paste(as.character(i),"-",as.character(network[j]))]] <- graph
}
}

In my article, I use the ggraph package to project the networks. Here, I resort to the sigmajs package (Coene 2020) for building interactive graphs. It relies on the Sigma javascript library, which is also used for the software GEPHI.

# Isolating the nodes and giving the proper names to the columns,
# to project with sigmajs
nodes <- graph_list[[1]] %>%
  activate(nodes) %>%
  as_tibble() %>%
  rename(id = Id) %>%
  mutate(label = paste0(Author_date,", ",Title,", ",Journal)) %>% 
  select(id,Year,Author,label,Title,Journal,size,color,x,y) 

edges <- graph_list[[1]] %>%
  activate(edges) %>%
  as_tibble() %>%
  rename(source = Source, target = Target, size = weight, color = color_edges) %>%
  mutate(id = 1:n()) %>% 
  select(id,source,target,color,size) %>% 
  as.data.table() 

# every identifier in character format for projection
nodes$id <- as.character(nodes$id)
edges$source <- as.character(edges$source)
edges$target <- as.character(edges$target)
edges$id <- as.character(edges$id)

# projecting the interactive graph
x <- sigmajs() %>% # initialise
      sg_nodes(nodes, id, label, size, color, x, y) %>% # add nodes
      sg_edges(edges, id, source, target, size, color) %>% # add edges
      sg_settings(drawLabels = FALSE, drawEdgeLabels = FALSE, 
                 defaultEdgeType = "curve", minNodeSize = 3, maxNodeSize = 20, 
                minEdgeSize = 0.5, maxEdgeSize = 3,
                 borderSize = 0.5, labelHoverBGColor = "node", singleHover = TRUE,
                 hideEdgesOnMove = TRUE, zoomMin = 0.4, zoomMax = 1.2) %>% 
      sg_neighbors()

frameWidget(x)

# Isolating the nodes and giving the proper names to the columns,
# to project with sigmajs
nodes <- graph_list[[2]] %>%
  activate(nodes) %>%
  as_tibble() %>%
  rename(id = Id) %>%
  mutate(label = paste0(Author_date,", ",Title,", ",Journal)) %>% 
  select(id,Year,Author,label,Title,Journal,size,color,x,y) 

edges <- graph_list[[2]] %>%
  activate(edges) %>%
  as_tibble() %>%
  rename(source = Source, target = Target, size = weight, color = color_edges) %>%
  mutate(id = 1:n()) %>% 
  select(id,source,target,color,size) %>% 
  as.data.table() 

nodes$id <- as.character(nodes$id)
edges$source <- as.character(edges$source)
edges$target <- as.character(edges$target)
edges$id <- as.character(edges$id)


x <- sigmajs() %>% # initialise
      sg_nodes(nodes, id, label, size, color, x, y) %>% # add nodes
      sg_settings(drawLabels = FALSE, drawEdgeLabels = FALSE, 
                 defaultEdgeType = "curve", minNodeSize = 2, maxNodeSize = 18, 
                 minEdgeSize = 0.5, maxEdgeSize = 2,
                 borderSize = 2, labelHoverBGColor = "node", singleHover = TRUE,
                 hideEdgesOnMove = TRUE, zoomMin = 0.4, zoomMax = 1.2) %>% 
      sg_edges(edges, id, source, target, size, color) %>% # add edges
      sg_neighbors()

frameWidget(x)

# Isolating the nodes and giving the proper names to the columns,
# to project with sigmajs
nodes <- graph_list[[3]] %>%
  activate(nodes) %>%
  as_tibble() %>%
  rename(id = Id) %>%
  mutate(label = paste0(Author_date,", ",Title,", ",Journal)) %>% 
  select(id,Year,Author,label,Title,Journal,size,color,x,y) 

edges <- graph_list[[3]] %>%
  activate(edges) %>%
  as_tibble() %>%
  rename(source = Source, target = Target, size = weight, color = color_edges) %>%
  mutate(id = 1:n()) %>% 
  select(id,source,target,color,size) %>% 
  as.data.table() 
nodes$id <- as.character(nodes$id)
edges$source <- as.character(edges$source)
edges$target <- as.character(edges$target)
edges$id <- as.character(edges$id)


x <- sigmajs() %>% # initialise
      sg_nodes(nodes, id, label, size, color, x, y) %>% # add nodes
      sg_settings(drawLabels = FALSE, drawEdgeLabels = FALSE, 
                 defaultEdgeType = "curve", minNodeSize = 3, maxNodeSize = 20, 
                 minEdgeSize = 0.5, maxEdgeSize = 1.8,
                 borderSize = 2, labelHoverBGColor = "node", singleHover = TRUE,
                 hideEdgesOnMove = TRUE, zoomMin = 0.4, zoomMax = 1.2) %>% 
      sg_edges(edges, id, source, target, size, color) %>% # add edges
      sg_neighbors()

frameWidget(x)

# Isolating the nodes and giving the proper names to the columns,
# to project with sigmajs
nodes <- graph_list[[3]] %>%
  activate(nodes) %>%
  as_tibble() %>%
  rename(id = Id) %>%
  mutate(label = paste0(Author_date,", ",Title,", ",Journal)) %>% 
  select(id,Year,Author,label,Title,Journal,size,color,x,y) 

edges <- graph_list[[3]] %>%
  activate(edges) %>%
  as_tibble() %>%
  rename(source = Source, target = Target, size = weight, color = color_edges) %>%
  mutate(id = 1:n()) %>% 
  select(id,source,target,color,size) %>% 
  as.data.table() 

nodes$id <- as.character(nodes$id)
edges$source <- as.character(edges$source)
edges$target <- as.character(edges$target)
edges$id <- as.character(edges$id)


x <- sigmajs() %>% # initialise
      sg_nodes(nodes, id, label, size, color, x, y) %>% # add nodes
      sg_settings(drawLabels = FALSE, drawEdgeLabels = FALSE, 
                 defaultEdgeType = "curve", minNodeSize = 3, maxNodeSize = 18, 
                 minEdgeSize = 0.5, maxEdgeSize = 2,
                 borderSize = 2, labelHoverBGColor = "node", singleHover = TRUE,
                 hideEdgesOnMove = TRUE, zoomMin = 0.4, zoomMax = 1.2) %>% 
      sg_edges(edges, id, source, target, size, color) %>% # add edges
      sg_neighbors()

frameWidget(x)

Another package for building interactive networks

By way of conclusion, I would like to write some words on the building of interactive networks. The sigmajs package is great as it allows R users to resort to Sigma Javascript library, which is a really great tool for network visualisations. With very few manipulations, you can build an interactive graph with interesting features (displaying labels and seeing connected nodes when clicking on one node). However, it offers only limited possibilities, in comparisons to ggraph for instance, to change the visual features of your graph (for instance, putting a border line around each node, having a long label on several lines or changing edges and nodes opacity).

Consequently, I have also tried the ggiraph package (Gohel and Skintzos 2020), which relies on ggplot2. It allows me to do something similar to the ggraph visualisations of my article. It is more flexible, notably for the label. But the zoom function appears less fluid, and above all I was unable to reproduce sigmajs feature when clicking on the nodes.⁸

# Extracting the labl of most cited nodes (networkflow function)
top_nodes  <- top_nodes(graph_list[[1]], top_n = 5, top_n_per_com = 0, ordering_column = "size")

# building the graph like in ggraph, but using geom_point_interactive()
graph <- ggraph(graph_list[[1]], "manual", x = x, y = y) + 
  geom_edge_arc(aes(color = color_edges, width = weight), alpha = 0.4, strength = 0.2, show.legend = FALSE) +
  scale_edge_width_continuous(range = c(0.1,2)) +
  scale_edge_colour_identity() +
  geom_point_interactive(aes(x = x, y = y, tooltip = paste0(Author_date,"\nTitle: ",Title,"\nJournal: ",Journal), data_id = Com_ID, size = size, fill = color), pch = 21, alpha = 0.9, show.legend = FALSE) +
  scale_size_continuous(range = c(0.2,13)) +
  scale_fill_identity() +
  new_scale("size") +
  geom_text(data=top_nodes, aes(x=x, y=y, label = Author_date), size = 3, fontface="bold", alpha = 1, show.legend = FALSE) +
  scale_size_continuous(range = c(0.5,5)) +
  theme_void()

# transforming the graph in an interactive graph with ggiraph
x <- girafe(ggobj = graph)

x <- girafe_options(x = x,
                    opts_tooltip(opacity = .8, use_fill = TRUE),
                    opts_zoom(min = 0.8, max = 4),
                    sizingPolicy(defaultWidth = "100%", defaultHeight = "250px"),
                    opts_hover_inv(css = "opacity:0.1;"),
                    opts_hover(css = "opacity:1;r:5pt;"))

frameWidget(x)

References