2a. Test Main FrEDI Data #60
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ### https://github.com/actions/upload-artifact | |
| ### https://github.blog/changelog/2021-11-10-github-actions-input-types-for-manual-workflows/ | |
| ### https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows | |
| ### https://github.com/r-lib/actions/tree/v2/setup-r-dependencies | |
| ### https://docs.github.com/en/actions/using-jobs/using-conditions-to-control-job-execution | |
| ### https://github.com/marketplace/actions/download-workflow-artifact | |
| ### https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/storing-and-sharing-data-from-a-workflow | |
| ### For uploading artifacts: | |
| ### "path:" is the output path where Pandoc will write the compiled PDF. | |
| ### Note, this should be the same directory as the input paper.md | |
| name: 2a. Test Main FrEDI Data | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| workflow_id: | |
| type: string | |
| description: Enter the run ID for the workflow from which to retrieve the main FrEDI Data | |
| fredi_branch: | |
| type: string | |
| description: Which FrEDI Package branch do you want to use? | |
| default: "main" | |
| gen_test: | |
| type: choice | |
| description: Run general tests on main FrEDI data? | |
| required: true | |
| options: | |
| - no | |
| - yes | |
| figures: | |
| type: choice | |
| description: Create scaled impact figures from main FrEDI data? | |
| required: true | |
| options: | |
| - no | |
| - yes | |
| sector: | |
| type: string | |
| description: Which sectors do you want to run? | |
| default: "all" | |
| jobs: | |
| # download: | |
| # runs-on: ubuntu-latest | |
| # # needs: install | |
| # env: | |
| # GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} | |
| # steps: | |
| # ### Download tmp_sysdata.rda from 1. Compile Main FrEDI Data run | |
| # - name: Download all artifacts | |
| # id: download-artifact | |
| # uses: actions/download-artifact@v4 | |
| # with: | |
| # github-token: ${{ secrets.GITHUB_TOKEN }} | |
| # # github-token: ${{ GITHUB_PAT }} | |
| # run-id: ${{ inputs.workflow_id }} | |
| # path: ./data | |
| # - run: | | |
| # pwd | |
| # ls -R ./data/Data | |
| test_data: | |
| runs-on: ubuntu-latest | |
| # needs: download | |
| env: | |
| GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} | |
| name: Process Data | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| # echo "$${{ github.ref_name }} ${{ inputs.workflow_id }} ${{ inputs.fredi_branch }} ${{ inputs.gen_test }} ${{ inputs.figures }}" | |
| - name: Send input status | |
| run: | | |
| echo "$${{ github.ref_name }} ${{ inputs.fredi_branch }} ${{ inputs.gen_test }} ${{ inputs.figures }}" | |
| - name: Setup R | |
| uses: r-lib/actions/setup-r@v2 | |
| - name: Setup R package dependencies | |
| uses: r-lib/actions/setup-r-dependencies@v2 | |
| with: | |
| cache: true | |
| cache-version: 1 | |
| packages: | | |
| any::tidyverse | |
| any::ggpubr | |
| any::openxlsx | |
| any::devtools | |
| any::admisc | |
| any::zoo | |
| - name: Install FrEDI | |
| run: | | |
| Rscript -e ' | |
| devtools::install_github( | |
| repo = "https://github.com/USEPA/FrEDI", | |
| ref = "${{ github.event.inputs.fredi_branch }}", | |
| dependencies = FALSE, | |
| upgrade = "never", | |
| force = TRUE, | |
| type = "source" | |
| )' | |
| ### Download tmp_sysdata.rda from 1. Compile Main FrEDI Data run | |
| - name: Download all artifacts | |
| id: download-artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| # github-token: ${{ GITHUB_PAT }} | |
| run-id: ${{ inputs.workflow_id }} | |
| path: ./data | |
| # - run: | | |
| # pwd | |
| # ls -R ./data/Data | |
| - name: Run Tests on Main FrEDI Data | |
| run: | | |
| Rscript -e ' | |
| ### Libraries | |
| require(tidyverse) | |
| require(devtools) | |
| ### Arguments | |
| doGenTests <- "${{ inputs.gen_test }}" %in% c("true", "yes") | |
| doFigures <- "${{ inputs.figures }}" %in% c("true", "yes") | |
| doTests <- doGenTests | doFigures | |
| sectors0 <- "${{ inputs.sector }}" | |
| ### Directories for project, scripts, data, and artifacts | |
| projectDir <- "." | |
| ### Load all code | |
| projectDir |> devtools::load_all() | |
| ### Create directory for outputs | |
| ### - Filename for general tests | |
| ### - Filename for list of tests | |
| outDir <- projectDir |> file.path("data_tests") | |
| doOutDir <- !(outDir |> dir.exists()) | |
| if(doOutDir) outDir |> dir.create() | |
| ### Output paths | |
| testFile <- "configTestResults" |> paste0(".", "xlsx") | |
| listFile <- "testList" |> paste0(".", "rda") | |
| listPath <- outDir |> file.path(listFile) | |
| ### Load Scripts | |
| scriptDir <- projectDir |> file.path("scripts") | |
| scriptName <- c("testFrediData") |> paste0(".", "R") | |
| scriptPath <- scriptDir |> file.path(scriptName) | |
| scriptPath |> source() | |
| # for(path_i in scriptPath){path_i |> source()} | |
| ### Load Data | |
| dataDir <- projectDir |> file.path("data") | |
| artDir <- dataDir |> file.path("Data") | |
| # getwd() |> print() | |
| # "." |> list.files() |> print(); | |
| # dataDir |> list.files() |> print(); | |
| # artDir |> list.files() |> print(); | |
| ### Artifact paths | |
| artName <- "tmp_sysdata.rda" | |
| artPath <- artDir |> file.path(artName) | |
| artExists <- artPath |> file.exists() | |
| artExists |> print() | |
| ### Whether to run the test | |
| runTests <- doTests & artExists | |
| ### Modify sectors | |
| sectors0 |> print() | |
| sectors0 <- sectors0 |> trimws() | |
| sectors0 <- sectors0 |> str_replace_all(" ", "") | |
| sectors0 <- sectors0 |> trimws() | |
| sectors0 <- sectors0 |> str_split(pattern=",") |> unlist() | |
| sectors0 |> print() | |
| ### Run general test | |
| if(runTests) { | |
| ### Load data | |
| artPath |> load() | |
| ### Run tests | |
| testList <- projectDir |> testFrediData( | |
| dataList = rDataList , ### List with data (e.g., rDataList) | |
| general = doGenTests, ### Whether to run the general test | |
| figures = doFigures , ### Whether to create scaled impact figures | |
| sectors = sectors0 , ### Which sector to run | |
| outDir = outDir , ### Where to save output files | |
| testFile = testFile ### Name of Excel file for testing results | |
| ) ### End testFrediData | |
| ### Save data | |
| testList |> save(file=listPath) | |
| outDir |> list.files() |> print() | |
| } ### End if(doTests) | |
| ' | |
| - name: Upload Tests | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: Tests | |
| path: | | |
| ./data_tests/* | |