diff --git a/TODO.md b/TODO.md index 60ade4c..bda7fc2 100644 --- a/TODO.md +++ b/TODO.md @@ -12,7 +12,6 @@ the [Unjournal coda.io project management website](https://coda.io/d/Project-Man # Descriptive work [x] What are we evaluating? How many papers in total? In what subfields? What does our “funnel” (selection process of candidate papers) look like? -(DR @DHJ -- can you point to the place to find these?) [ ] How’s our process? How many papers per month? Turnaround times? Numbers of evaluators per paper? diff --git a/code/import-unjournal-data.R b/code/import-unjournal-data.R index 972f0c1..2379527 100644 --- a/code/import-unjournal-data.R +++ b/code/import-unjournal-data.R @@ -27,14 +27,14 @@ base_id <- "applDG6ifmUmeEJ7j" # new ID to cover "UJ - research & core members" pub_records <- air_select(base = base_id, table = "crucial_rsx") all_pub_records <- pub_records -# 100 is the maximum length returned -while(nrow(pub_records) == 100) { - # Get the ID of the last record in the list - offset <- get_offset(pub_records) - # Fetch the next records, starting after this ID - pub_records <- air_select(base = base_id, table = "crucial_rsx", offset = offset) - # Append the records to the df +offset <- get_offset(pub_records) + +# the offset is only returned while there are more records +while(! is.null(offset)) { + pub_records <- air_select(base = base_id, table = "crucial_rsx", + offset = offset) all_pub_records <- bind_rows(all_pub_records, pub_records) + offset <- get_offset(pub_records) } rm(pub_records)