From 20267a6a5a49cac4a171131cd4268a90311e8e4f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 20 Apr 2021 13:30:35 +0000 Subject: [PATCH 01/15] Dynamically pull repo information for HPC lessons --- .github/workflows/build-and-deploy.yml | 9 ++ Makefile | 7 + R/README.md | 1 + R/help_wanted_issues.R | 182 +++++++++++++++++++++++++ R/hpc-carpentry_lessons.R | 76 +++++++++++ R/utils.R | 170 +++++++++++++++++++++++ pages/community-lessons.md | 4 +- 7 files changed, 447 insertions(+), 2 deletions(-) create mode 100644 R/README.md create mode 100644 R/help_wanted_issues.R create mode 100644 R/hpc-carpentry_lessons.R create mode 100644 R/utils.R diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index ea839cb47..24dd13abb 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -3,6 +3,7 @@ name: check, build, deploy hpc-carpentry.org on: push: branches: main + pull_request: [] jobs: build-website: @@ -40,6 +41,14 @@ jobs: curl --remote-name-all https://feeds.carpentries.org/all_instructors_by_airport.geojson &&\ cp all_instructors_by_airport.geojson files/. + - name: Create repository data files + uses: eessi/github-action-eessi@v1 + shell: bash + env: + GITHUB_PAT: ${{ secrets.JEKYLL_PAT }} + run: | + module load R + make data # - name: Create PDF file of some pages # uses: docker://pandoc/latex:latest diff --git a/Makefile b/Makefile index d83c8eb66..ad8518edf 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,13 @@ site : install : bundle install +## lesson data : pull lesson data from GitHub +# (requires GitHub PAT provided via GITHUB_PAT env var) +data: + R -q -e "source('R/hpc-carpentry_lessons.R')" +## help-wanted: list of issues that have the label "help wanted" + R -q -e "source('R/help_wanted_issues.R')" + #------------------------------------------------------------------------------- ## clean : clean up junk files. diff --git a/R/README.md b/R/README.md new file mode 100644 index 000000000..1edb1d8ba --- /dev/null +++ b/R/README.md @@ -0,0 +1 @@ +The files in this directory are based off of https://github.com/carpentries/feeds.carpentries.org (also MIT licence). If they require an update you should check there first. diff --git a/R/help_wanted_issues.R b/R/help_wanted_issues.R new file mode 100644 index 000000000..1905dd91f --- /dev/null +++ b/R/help_wanted_issues.R @@ -0,0 +1,182 @@ +source("R/utils.R") + +new_tbl_github_issues <- function(url = character(0), + title = character(0), + type = character(0), + labels = character(0), + label_colors = character(0), + font_colors = character(0), + created_at = character(0), + updated_at = character(0), + empty = FALSE + ) { + + stopifnot(rlang::is_scalar_logical(empty)) + + if (empty) { + stopifnot(rlang::is_character(url, n = 0L)) + stopifnot(rlang::is_character(title, n = 0L)) + stopifnot(rlang::is_character(type, n = 0L)) + stopifnot(rlang::is_character(labels, n = 0L)) + stopifnot(rlang::is_character(label_colors, n = 0L)) + stopifnot(rlang::is_character(font_colors, n = 0L)) + stopifnot(rlang::is_character(created_at, n = 0L)) + stopifnot(rlang::is_character(updated_at, n = 0L)) + } else { + stopifnot(rlang::is_scalar_character(url)) + stopifnot(rlang::is_scalar_character(title)) + stopifnot(rlang::is_scalar_character(type)) + stopifnot(rlang::is_character(labels)) + stopifnot(rlang::is_character(label_colors)) + stopifnot(rlang::is_character(font_colors)) + stopifnot(rlang::is_scalar_character(created_at)) + stopifnot(rlang::is_scalar_character(updated_at)) + } + + tibble::tibble( + url, + title, + type, + labels, + label_colors, + font_colors, + created_at, + updated_at + ) + +} + +get_gh_issues_raw <- function(owner, repo, labels) { + if (!is.null(labels)) { + stopifnot(identical(length(labels), 1L)) + } + gh::gh( + "GET /repos/:owner/:repo/issues", + owner = owner, + repo = repo, + labels = labels + ) +} + +extract_issue_info <- function(issues) { + + if (identical(length(issues), 0L)) { + return(new_tbl_github_issues(empty = TRUE)) + } + + issues %>% + purrr::map_df(function(.x) { + new_tbl_github_issues( + url = .x$html_url, + title = .x$title, + type = dplyr::case_when( + grepl("/pull/[0-9]+$", .x$html_url) ~ "PR", + TRUE ~ "issue" + ), + labels = purrr::map_chr(.x$labels, "name") %>% + paste(., collapse = ","), + label_colors = purrr::map_chr(.x$labels, "color") %>% + paste0("#", ., collapse = ","), + font_colors = purrr::map_chr(.x$labels, "color") %>% + paste0("#", .) %>% + font_color(.) %>% + paste(., collapse = ","), + created_at = .x$created_at, + updated_at = .x$updated_at + ) + }) +} + +get_gh_issues <- function(owner, repo, labels) { + get_gh_issues_raw(owner, repo, labels) %>% + extract_issue_info() %>% + dplyr::mutate( + org = owner, + repo = repo, + full_repo = paste0(owner, "/", repo) + ) +} + +keep_opted_in <- function(orgs) { + + # at_opted_in <- airtabler::airtable( + # base = "appeZJGnGremE1MYm", + # tables = "Repositories" + # ) + + # opted_in <- at_opted_in$Repositories$select_all() %>% + # dplyr::mutate(lesson_program = tolower(lesson_program)) + # + # dplyr::left_join( + # orgs, opted_in, + # by = c( + # "carpentries_org" = "lesson_program", + # "repo" = "repository" + # ) + # ) %>% + # dplyr::filter(display_help_wanted) + # message(sQuote(orgs)) + orgs +} + +keep_other_repos <- function(orgs) { + other_repos <- tibble::tribble( + ~carpentries_org, ~repo, + "carpentries-incubator", "hpc-intro", + "hpc-carpentry", "hpc-carpentry.github.io", + ) + + dplyr::inner_join( + orgs, other_repos, + by = c("carpentries_org", "repo") + ) +} + + +list_organizations <- c( + "HPC Carpentry" = "hpc-carpentry" + # would like to include the incubator but need to filter the repo according to topic + # "The Carpentries Incubator" = "carpentries-incubator" +) + +list_help_wanted <- purrr::imap_dfr( + list_organizations, + function(.x, .y) { + orgs <- get_list_repos( + .x, ignore_archived = TRUE, + ignore_pattern = "^\\d{4}-\\d{2}-\\d{2}" + ) + + lessons <- orgs %>% + keep_opted_in() + + other_repos <- orgs %>% + keep_other_repos() + + dplyr::bind_rows( + lessons, + other_repos + ) %>% + dplyr::distinct(carpentries_org, repo, .keep_all = TRUE) %>% + purrr::pmap_df(function(carpentries_org, repo, description, ...) { + message(" repo: ", repo, appendLF = FALSE) + res <- purrr::map_dfr( + c("help wanted", "good first issue"), + ~ get_gh_issues( + owner = carpentries_org, repo = repo, labels = .x + ) + ) %>% + dplyr::distinct(url, .keep_all = TRUE) + message(" -- n issues: ", nrow(res)) + res %>% + dplyr::mutate( + description = description, + ## remove GitHub emoji from repo description + clean_description = gsub(":([a-z0-9_]+):", "", description), + org_name = .y) + }) + } +) + + +jsonlite::write_json(list_help_wanted, "_data/help_wanted_issues.json") diff --git a/R/hpc-carpentry_lessons.R b/R/hpc-carpentry_lessons.R new file mode 100644 index 000000000..a8ead1c57 --- /dev/null +++ b/R/hpc-carpentry_lessons.R @@ -0,0 +1,76 @@ +source("R/utils.R") + +LIFE_CYCLE_TAGS <- c("pre-alpha", "alpha", "beta", "stable") +# The tags below will be filtered out in the json +COMMON_TAGS <- c( + "carpentries", + "carpentries-incubator", + "carpentries-lesson", + "carpentryconnect", + "data-carpentry", + "datacarpentry", + "education", + "lesson" +) + +check_missing_repo_info <- function(.d, field) { + if (any(!nzchar(.d[[field]]))) { + paste0( + "Missing repo ", sQuote(field), " for: \n", + paste0(" - ", .d$repo_url[!nzchar(.d[[field]])], collapse = "\n"), + "\n" + ) + } +} + +check_repo_info <- function(.d, fields) { + tryCatch({ + out <- purrr::map( + fields, ~ check_missing_repo_info(.d, .) + ) + msgs <- purrr::keep(out, ~ !is.null(.)) + + if (length(msgs)) { + stop(msgs, call. = FALSE) + } + + cli::cli_alert_success("No issues detected!") + }, + error = function(err) { + stop(err$message, call. = FALSE) + }) +} + +make_community_lessons_feed <- function(path, ...) { + + carp_inc <- get_org_topics("carpentries-incubator") + hpc_carp <- get_org_topics("hpc-carpentry") + + res <- dplyr::bind_rows(carp_inc, hpc_carp) %>% + dplyr::select(-private) %>% + dplyr::filter(grepl("hpc-carpentry", github_topics)) %>% + dplyr::filter(grepl("lesson", github_topics)) %>% + extract_tag( + life_cycle_tag, + LIFE_CYCLE_TAGS, + approach = "include", + allow_multiple = FALSE, + allow_empty = FALSE + ) %>% + extract_tag( + lesson_tags, + COMMON_TAGS, + approach = "exclude", + allow_multiple = TRUE, + allow_empty = TRUE + ) + + ## checks + check_repo_info(res, c("description", "rendered_site")) + + res %>% + jsonlite::write_json(path = path) + +} + +make_community_lessons_feed("_data/hpc_lessons.json") diff --git a/R/utils.R b/R/utils.R new file mode 100644 index 000000000..47004cff5 --- /dev/null +++ b/R/utils.R @@ -0,0 +1,170 @@ +library(gh) +library(jsonlite) +library(purrr) + +`%<<%` <- function(x, y) { + if (identical(length(x), 0L)) return(y) + if (is.null(x) || identical(x, "") || + is.na(x)) return(y) + x +} + +use_pat <- function() { + if (Sys.getenv("GITHUB_PAT") != "" || Sys.getenv("GITHUB_TOKEN") != "") { + cli::cli_alert_success("Using GitHub token!") + } else { + cli::cli_alert_danger("No GitHub token detected.") + } +} +use_pat() + + +get_list_repos <- function(org, ignore_archived = FALSE, + ignore_pattern = NULL, ...) { + + init_res <- gh::gh("GET /orgs/:org/repos", org = org) + res <- list() + test <- TRUE + i <- 1 + + while (test) { + message("Getting page: ", i, " for ", sQuote(org)) + res <- append(res, init_res) + + init_res <- tryCatch({ + gh::gh_next(init_res) + }, + error = function(e) { + test <<- FALSE + NULL + }) + i <- i+1 + } + # message(sQuote(res)) + res <- purrr::map_df(res, function(.x) { + list( + carpentries_org = tolower(.x$owner$login) %<<% "", + repo = .x$name, + repo_url = .x$html_url, + full_name = .x$full_name, + description = .x$description %<<% "", + rendered_site = .x$homepage %<<% "", + private = .x$private, + archived = .x$archived + ) + }) %>% + dplyr::filter( + !private, + tolower(carpentries_org) == tolower(org) + ) + + if (ignore_archived) { + res <- res %>% + dplyr::filter(!archived) + } + + if (!is.null(ignore_pattern)) { + res <- res %>% + dplyr::filter(!grepl(ignore_pattern, repo, ...)) + } + + res %>% + dplyr::select(-archived) +} + + +font_color <- function(hexcode) { + rgb <- colorspace::hex2RGB(hexcode) + rgbR <- rgb@coords[, "R"] + rgbG <- rgb@coords[, "G"] + rgbB <- rgb@coords[, "B"] + luma <- ((0.299 * rgbR) + (0.587 * rgbG) + (0.114 * rgbB)) + res <- rep("#ffffff", length(hexcode)) + res[luma > .5] <- "#222222" + res +} + + +get_github_topics <- function(owner, repo) { + res <- gh::gh( + "GET /repos/:owner/:repo/topics", + owner = owner, repo = repo, + .send_headers = c("Accept" = "application/vnd.github.mercy-preview+json") + ) + + purrr::map_chr(res[["names"]], ~ .) +} + +get_org_topics <- function(org) { + + get_list_repos(org) %>% + dplyr::filter( + !private, + carpentries_org == org + ) %>% + dplyr::mutate( + github_topics = purrr::pmap(., function(carpentries_org, repo, ...) { + get_github_topics(carpentries_org, repo) %<<% "" + }) + ) +} + + +##' @param data the data frame that contains the column `github_topics` from +##' which the tags should be extracted +##' @param new_col_name name of the new column that will contain the extracted +##' tags +##' @param dict the dictionary of tags (as a character vector) against which the +##' content of the `github_topics` column will be compared +##' @param approach when set to `include` the tag(s) that match(es) the content +##' of the vector specified in `dict` will be extracted to create the content +##' of the new column; when set to `exclude` the tag(s) that match(es) the +##' content of the vector specified in `dict` will be excluded to create the +##' content of the new column. For instance if the column `github_topics` has +##' the values `c("tag1", "tag2")` and `dict` is `"tag1"`, the new column will +##' have the value `tag1` when using `include` and `tag2` when using +##' `exclude`. +##' @param allow_multiple can the resulting new column contain more than one +##' tag? +##' @param allow_empty can the resulting new column contain an empty value? +extract_tag <- function(data, + new_col_name, + dict, + approach = c("include", "exclude"), + allow_multiple, + allow_empty + ) { + + approach <- match.arg(approach) + + if (identical(approach, "include")) { + f1 <- intersect + f2 <- setdiff + } else if (identical(approach, "exclude")) { + f1 <- setdiff + f2 <- intersect + } else { + stop("invalid value for approach") + } + + data %>% + dplyr::mutate(!!rlang::quo_name(rlang::enquo(new_col_name)) := purrr::pmap(., + function(github_topics, full_name, ...) { + extracted_tag <- f1(github_topics, dict) + if ((!allow_multiple) && length(extracted_tag) > 1) { + stop("More than one tag detected for: ", full_name) + } + if (length(extracted_tag) == 0) { + if (! allow_empty) { + stop("No tag found among (", paste(dict, collapse = ", "), ") ", + "for repo: ", full_name) + } + return("") + } + extracted_tag + })) %>% + dplyr::mutate(github_topics = purrr::pmap(., function(github_topics, ...) { + f2(github_topics, dict) + })) + +} diff --git a/pages/community-lessons.md b/pages/community-lessons.md index cb1339bed..b9163feb8 100644 --- a/pages/community-lessons.md +++ b/pages/community-lessons.md @@ -61,9 +61,9 @@ Please read the information on our [Development of Lessons page](/involved-lesso if you would like to contribute to the development of a lesson already present in the HPC Carpentry Incubator. -## Lessons in the HPC Carpentry Incubator +## Lessons in the HPC Carpentry -{% assign lesson_list = site.data.community_lessons | where: "hpc_carpentry_org","hpc-carpentry-incubator" %} +{% assign lesson_list = site.data.hpc_lessons %} {% include lesson_table %}
From 526c20bbe0d62cffb811d18f255e6a6799c55e12 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 20 Apr 2021 14:04:48 +0000 Subject: [PATCH 02/15] Update to use latest json data --- _data/community_lessons.json | 81 ------------------------------------ pages/community-lessons.md | 4 +- 2 files changed, 2 insertions(+), 83 deletions(-) delete mode 100644 _data/community_lessons.json diff --git a/_data/community_lessons.json b/_data/community_lessons.json deleted file mode 100644 index bea44fc0a..000000000 --- a/_data/community_lessons.json +++ /dev/null @@ -1,81 +0,0 @@ -[ - { - "hpc_carpentry_org": "hpc-carpentry-incubator", - "repo": "hpc-intro", - "repo_url": "https://github.com/hpc-carpentry/hpc-intro", - "full_name": "hpc-carpentry/hpc-intro", - "description": "Intro to HPC lesson materials", - "rendered_site": "http://hpc-carpentry.org/hpc-intro/", - "github_topics": [ - "hpc-carpentry-incubator", - "lesson" - ], - "life_cycle_tag": [ - "alpha" - ], - "lesson_tags": [ - "HPC", - "english" - ] - }, - { - "hpc_carpentry_org": "hpc-carpentry-incubator", - "repo": "hpc-shell", - "repo_url": "https://github.com/hpc-carpentry/hpc-shell", - "full_name": "hpc-carpentry/hpc-shell", - "description": "Materials to teach terminal fundamentals for HPC users", - "rendered_site": "http://hpc-carpentry.org/hpc-shell/", - "github_topics": [ - "hpc-carpentry-incubator", - "lesson" - ], - "life_cycle_tag": [ - "alpha" - ], - "lesson_tags": [ - "HPC", - "english", - "shell" - ] - }, - { - "hpc_carpentry_org": "hpc-carpentry-incubator", - "repo": "hpc-python", - "repo_url": "https://github.com/hpc-carpentry/hpc-python", - "full_name": "hpc-carpentry/hpc-python", - "description": "HPC Python lesson materials", - "rendered_site": "http://hpc-carpentry.org/hpc-python/", - "github_topics": [ - "hpc-carpentry-incubator", - "lesson" - ], - "life_cycle_tag": [ - "alpha" - ], - "lesson_tags": [ - "HPC", - "english", - "python" - ] - }, - { - "hpc_carpentry_org": "hpc-carpentry-incubator", - "repo": "hpc-chapel", - "repo_url": "https://github.com/hpc-carpentry/hpc-chapel", - "full_name": "hpc-carpentry/hpc-chapel", - "description": "HPC Chapel lesson materials", - "rendered_site": "http://hpc-carpentry.org/hpc-chapel/", - "github_topics": [ - "hpc-carpentry-incubator", - "lesson" - ], - "life_cycle_tag": [ - "alpha" - ], - "lesson_tags": [ - "HPC", - "english", - "Chapel" - ] - } -] diff --git a/pages/community-lessons.md b/pages/community-lessons.md index b9163feb8..011ea966b 100644 --- a/pages/community-lessons.md +++ b/pages/community-lessons.md @@ -101,13 +101,13 @@ submit it first to the Incubator. ## List of Community Developed Lessons by Topic -{% assign lesson_topics = site.data.community_lessons | map: "lesson_tags" | uniq | sort%} +{% assign lesson_topics = site.data.hpc_lessons | map: "lesson_tags" | uniq | sort%} {% for t in lesson_topics %} ### {{ t | capitalize | replace: "-", " "}} -{% assign lesson_with_tag = site.data.community_lessons | where_exp: "item", "item.lesson_tags contains t" %} +{% assign lesson_with_tag = site.data.hpc_lessons | where_exp: "item", "item.lesson_tags contains t" %} {% for l in lesson_with_tag %} - [{{l.description}}](#{{ l.description | slugify }}) {{l.carpentries_org}} From c7601455986da598e4499be8174e46a967040ac2 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 20 Apr 2021 16:11:46 +0200 Subject: [PATCH 03/15] Fix action --- .github/workflows/build-and-deploy.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index 24dd13abb..57239d8ea 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -3,7 +3,9 @@ name: check, build, deploy hpc-carpentry.org on: push: branches: main - pull_request: [] + pull_request: + branches: + - main jobs: build-website: From fef50ed41e64ddaa54970a55dcb1e6f5e6aca2f3 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 20 Apr 2021 16:16:15 +0200 Subject: [PATCH 04/15] Fix CI --- .github/workflows/build-and-deploy.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index 57239d8ea..29fa1d721 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -38,13 +38,10 @@ jobs: yaml-lint -q -n $(find _posts -regex ".*.md\|.*html") &&\ yaml-lint -q -n $(find pages -regex ".*.md\|.*html") - - name: Get GeoJSON for instructor map - run: | - curl --remote-name-all https://feeds.carpentries.org/all_instructors_by_airport.geojson &&\ - cp all_instructors_by_airport.geojson files/. + - name: Setup EESSI to give us an R module + uses: eessi/github-action-eessi@v1 - name: Create repository data files - uses: eessi/github-action-eessi@v1 shell: bash env: GITHUB_PAT: ${{ secrets.JEKYLL_PAT }} From 45747e243603ac93c80736f4214e42f53258c546 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 20 Apr 2021 16:26:02 +0200 Subject: [PATCH 05/15] Update build-and-deploy.yml --- .github/workflows/build-and-deploy.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build-and-deploy.yml b/.github/workflows/build-and-deploy.yml index 29fa1d721..d46f4da2e 100644 --- a/.github/workflows/build-and-deploy.yml +++ b/.github/workflows/build-and-deploy.yml @@ -3,9 +3,6 @@ name: check, build, deploy hpc-carpentry.org on: push: branches: main - pull_request: - branches: - - main jobs: build-website: From 1216dd305020a639a2752a6370a945126a52f7e9 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 21 Apr 2021 09:19:16 +0200 Subject: [PATCH 06/15] Add hpc-carpentry as tag to filter --- R/hpc-carpentry_lessons.R | 1 + 1 file changed, 1 insertion(+) diff --git a/R/hpc-carpentry_lessons.R b/R/hpc-carpentry_lessons.R index a8ead1c57..45277e836 100644 --- a/R/hpc-carpentry_lessons.R +++ b/R/hpc-carpentry_lessons.R @@ -8,6 +8,7 @@ COMMON_TAGS <- c( "carpentries-lesson", "carpentryconnect", "data-carpentry", + "hpc-carpentry", "datacarpentry", "education", "lesson" From a70358a01df0dcba9ece640aebd83ff4095133fc Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 21 Apr 2021 11:08:10 +0200 Subject: [PATCH 07/15] Fix broken link Remove use of json plugin (for now) Synch help-wanted page with upstream --- Gemfile | 2 +- _config.yml | 8 ++---- pages/help-wanted-issues.md | 50 +++++++++++++++++++++---------------- pages/teach.md | 10 ++++++++ 4 files changed, 42 insertions(+), 28 deletions(-) create mode 100644 pages/teach.md diff --git a/Gemfile b/Gemfile index 4b5ecef9f..83d687db0 100644 --- a/Gemfile +++ b/Gemfile @@ -4,5 +4,5 @@ gem 'faraday', '0.17.3' group :jekyll_plugins do gem 'github-pages' - gem 'jekyll-get-json', "~> 0.0.2" +# gem 'jekyll-get-json', "~> 0.0.2" end diff --git a/_config.yml b/_config.yml index 9f7ec871b..1ececc405 100644 --- a/_config.yml +++ b/_config.yml @@ -119,14 +119,10 @@ exclude: plugins: - jekyll-paginate - jekyll-redirect-from - - jekyll-get-json +# - jekyll-get-json ## external data sources -jekyll_get_json: -# - data: community_lessons -# json: 'https://feeds.carpentries.org/community_lessons.json' - - data: help_wanted_issues - json: 'https://feeds.carpentries.org/help_wanted_issues.json' +# jekyll_get_json: # - data: all_badged_people # json: 'https://feeds.carpentries.org/all_badged_people.json' # - data: badges_stats diff --git a/pages/help-wanted-issues.md b/pages/help-wanted-issues.md index 76313ad82..c0f4c71b1 100644 --- a/pages/help-wanted-issues.md +++ b/pages/help-wanted-issues.md @@ -20,18 +20,27 @@ permalink: "/help-wanted-issues/" Information for Lesson Maintainers -{% assign help_wanted = site.data.help_wanted_issues | where: "org_name", "The Carpentries Incubator" %} +{% assign help_wanted = site.data.help_wanted_issues %} -{% assign hpc_repos = site.data.lessons %} +{% comment %} +This was a way to get all the organizations automatically, but probably better to curate ordering {% assign orgs = help_wanted | map: "org_name" | uniq %} +{% endcomment %} -{% for each_repo in hpc_repos %} +{% assign orgs = "The Carpentries Incubator, HPC Carpentry" | split: ", " %} -{% assign org_repos = help_wanted | where: "repo", each_repo.repo %} + +{% for each_org in orgs %} + +{% assign org_repos = help_wanted | where: "org_name", each_org %} {% assign grouped_org_repos = org_repos | group_by: "full_repo" %} +{% if org_repos.size > 0 %} + +

{{ each_org }}

+ {% for r in grouped_org_repos %} {% assign repo_desc = r.items[0].clean_description %} @@ -54,7 +63,7 @@ permalink: "/help-wanted-issues/"
  • {{ i.title}} -{% for l in labels %}{{ l }}{% endfor %} +{% for l in labels %}{{ l }} {% endfor %}

    Type: {{ i.type }} @@ -65,20 +74,19 @@ Updated:

    {{ each_org }}

    - -{% for each_repo in hpc_repos %} - -{% assign org_repos = help_wanted | where: "repo", each_repo.repo %} +{% assign org_repos = help_wanted | where: "org_name", each_org %} {% assign grouped_org_repos = org_repos | group_by: "full_repo" %} +{% if grouped_org_repos.size > 0 %} +

    {{ each_org }}

    +
      {% for r in grouped_org_repos %} @@ -110,14 +117,13 @@ issues on your lesson repository][handbook-github-labels]. {% assign title = repo_desc %} {% endif %} +
    • {{ title }}
      • {% endfor %}
    - -{% endfor %} - +{% endif %} {% endfor %}
    @@ -126,4 +132,6 @@ issues on your lesson repository][handbook-github-labels].
    + + [handbook-github-labels]: https://docs.carpentries.org/topic_folders/maintainers/github_labels.html diff --git a/pages/teach.md b/pages/teach.md new file mode 100644 index 000000000..71f11cd4c --- /dev/null +++ b/pages/teach.md @@ -0,0 +1,10 @@ +--- +permalink: /teach/ +redirect_to: "https://carpentries.org/teach/" +layout: redirect +--- + +{% comment %} +friendly URL to provide a redirect to the Teaching with the +Carpentries page +{% endcomment %} From 75f2b93175980cbef771d13f7c86a9f027abfe6c Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 21 Apr 2021 13:28:55 +0000 Subject: [PATCH 08/15] Add a whitelist to Help Wanted --- R/help_wanted_issues.R | 44 +++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/R/help_wanted_issues.R b/R/help_wanted_issues.R index 1905dd91f..145b4da47 100644 --- a/R/help_wanted_issues.R +++ b/R/help_wanted_issues.R @@ -99,30 +99,27 @@ get_gh_issues <- function(owner, repo, labels) { keep_opted_in <- function(orgs) { - # at_opted_in <- airtabler::airtable( - # base = "appeZJGnGremE1MYm", - # tables = "Repositories" - # ) - - # opted_in <- at_opted_in$Repositories$select_all() %>% - # dplyr::mutate(lesson_program = tolower(lesson_program)) - # - # dplyr::left_join( - # orgs, opted_in, - # by = c( - # "carpentries_org" = "lesson_program", - # "repo" = "repository" - # ) - # ) %>% - # dplyr::filter(display_help_wanted) - # message(sQuote(orgs)) - orgs + # Repos that wish to opt in should be manually added here + opt_in_repos <- tibble::tribble( + ~carpentries_org, ~repo, + "carpentries-incubator", "hpc-intro", + ) + + dplyr::inner_join( + orgs, opt_in_repos, + by = c("carpentries_org", "repo") + ) +} + +keep_hpc_carpentry_repos <- function(orgs) { + dplyr::filter( + orgs, carpentries_org == "hpc-carpentry" + ) } keep_other_repos <- function(orgs) { other_repos <- tibble::tribble( ~carpentries_org, ~repo, - "carpentries-incubator", "hpc-intro", "hpc-carpentry", "hpc-carpentry.github.io", ) @@ -134,9 +131,8 @@ keep_other_repos <- function(orgs) { list_organizations <- c( - "HPC Carpentry" = "hpc-carpentry" - # would like to include the incubator but need to filter the repo according to topic - # "The Carpentries Incubator" = "carpentries-incubator" + "HPC Carpentry" = "hpc-carpentry", + "The Carpentries Incubator" = "carpentries-incubator" ) list_help_wanted <- purrr::imap_dfr( @@ -150,11 +146,15 @@ list_help_wanted <- purrr::imap_dfr( lessons <- orgs %>% keep_opted_in() + hpc_carpentry_lessons <- orgs %>% + keep_hpc_carpentry() + other_repos <- orgs %>% keep_other_repos() dplyr::bind_rows( lessons, + hpc_carpentry_lessons, other_repos ) %>% dplyr::distinct(carpentries_org, repo, .keep_all = TRUE) %>% From 7bb1e52ab15a6684d93bb0bf23645e6fa3a0145a Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 21 Apr 2021 13:45:59 +0000 Subject: [PATCH 09/15] Remove unnecessary duplication --- R/help_wanted_issues.R | 27 +++++---------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/R/help_wanted_issues.R b/R/help_wanted_issues.R index 145b4da47..7fe34ff3d 100644 --- a/R/help_wanted_issues.R +++ b/R/help_wanted_issues.R @@ -97,20 +97,6 @@ get_gh_issues <- function(owner, repo, labels) { ) } -keep_opted_in <- function(orgs) { - - # Repos that wish to opt in should be manually added here - opt_in_repos <- tibble::tribble( - ~carpentries_org, ~repo, - "carpentries-incubator", "hpc-intro", - ) - - dplyr::inner_join( - orgs, opt_in_repos, - by = c("carpentries_org", "repo") - ) -} - keep_hpc_carpentry_repos <- function(orgs) { dplyr::filter( orgs, carpentries_org == "hpc-carpentry" @@ -118,9 +104,10 @@ keep_hpc_carpentry_repos <- function(orgs) { } keep_other_repos <- function(orgs) { + # Repos that wish to opt in should be manually added here other_repos <- tibble::tribble( ~carpentries_org, ~repo, - "hpc-carpentry", "hpc-carpentry.github.io", + "carpentries-incubator", "hpc-intro", ) dplyr::inner_join( @@ -143,18 +130,14 @@ list_help_wanted <- purrr::imap_dfr( ignore_pattern = "^\\d{4}-\\d{2}-\\d{2}" ) - lessons <- orgs %>% - keep_opted_in() - - hpc_carpentry_lessons <- orgs %>% - keep_hpc_carpentry() + hpc_carpentry_repos <- orgs %>% + keep_hpc_carpentry_repos() other_repos <- orgs %>% keep_other_repos() dplyr::bind_rows( - lessons, - hpc_carpentry_lessons, + hpc_carpentry_repos, other_repos ) %>% dplyr::distinct(carpentries_org, repo, .keep_all = TRUE) %>% From 59c6fd76f582e98d9ddf1d91020559401b972b56 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 21 Apr 2021 13:59:13 +0000 Subject: [PATCH 10/15] Address review comments --- Makefile | 4 ++-- _config.yml | 3 +++ {R => feeds}/README.md | 0 {R => feeds}/help_wanted_issues.R | 2 +- {R => feeds}/hpc-carpentry_lessons.R | 2 +- {R => feeds}/utils.R | 0 pages/community-lessons.md | 10 +++++----- 7 files changed, 12 insertions(+), 9 deletions(-) rename {R => feeds}/README.md (100%) rename {R => feeds}/help_wanted_issues.R (99%) rename {R => feeds}/hpc-carpentry_lessons.R (98%) rename {R => feeds}/utils.R (100%) diff --git a/Makefile b/Makefile index a7e705914..c7594341a 100644 --- a/Makefile +++ b/Makefile @@ -23,9 +23,9 @@ install : ## lesson data : pull lesson data from GitHub # (requires GitHub PAT provided via GITHUB_PAT env var) data: - R -q -e "source('R/hpc-carpentry_lessons.R')" + R -q -e "source('feeds/hpc-carpentry_lessons.R')" ## help-wanted: list of issues that have the label "help wanted" - R -q -e "source('R/help_wanted_issues.R')" + R -q -e "source('feeds/help_wanted_issues.R')" #------------------------------------------------------------------------------- diff --git a/_config.yml b/_config.yml index 1ececc405..4936bc351 100644 --- a/_config.yml +++ b/_config.yml @@ -111,6 +111,7 @@ exclude: - README.md - INSTALL.md - Makefile + - feeds/ - vendor/ # Build settings @@ -119,6 +120,8 @@ exclude: plugins: - jekyll-paginate - jekyll-redirect-from +# For HPC Carpentry, we generate the json files as part of a GitHub Action +# so we don't (currently) need this dependency # - jekyll-get-json ## external data sources diff --git a/R/README.md b/feeds/README.md similarity index 100% rename from R/README.md rename to feeds/README.md diff --git a/R/help_wanted_issues.R b/feeds/help_wanted_issues.R similarity index 99% rename from R/help_wanted_issues.R rename to feeds/help_wanted_issues.R index 7fe34ff3d..cb154f6e8 100644 --- a/R/help_wanted_issues.R +++ b/feeds/help_wanted_issues.R @@ -1,4 +1,4 @@ -source("R/utils.R") +source("feeds/utils.R") new_tbl_github_issues <- function(url = character(0), title = character(0), diff --git a/R/hpc-carpentry_lessons.R b/feeds/hpc-carpentry_lessons.R similarity index 98% rename from R/hpc-carpentry_lessons.R rename to feeds/hpc-carpentry_lessons.R index 45277e836..85fc061fd 100644 --- a/R/hpc-carpentry_lessons.R +++ b/feeds/hpc-carpentry_lessons.R @@ -1,4 +1,4 @@ -source("R/utils.R") +source("feeds/utils.R") LIFE_CYCLE_TAGS <- c("pre-alpha", "alpha", "beta", "stable") # The tags below will be filtered out in the json diff --git a/R/utils.R b/feeds/utils.R similarity index 100% rename from R/utils.R rename to feeds/utils.R diff --git a/pages/community-lessons.md b/pages/community-lessons.md index e851d9f17..4abf0ea17 100644 --- a/pages/community-lessons.md +++ b/pages/community-lessons.md @@ -24,7 +24,7 @@ The HPC Carpentry community is committed to a collaborative and open process for lesson development and to sharing teaching materials. We provide three avenues for community members to share lesson materials - **The Carpentries Incubator**, **HPC Carpentry** and our -**HPC Carpentry Listings**. +**HPC Listings**. [The Carpentries Incubator](#the-carpentries-incubator) is for: * Collaborative lesson development (from conceptual to stable lessons). @@ -36,7 +36,7 @@ materials - **The Carpentries Incubator**, **HPC Carpentry** and our * Getting peer-review on the content of the lesson in the way traditional journal peer-review wouldn’t be able to provide. -[HPC Carpentry Listings](#hpc-carpentry-listings) is for: +[HPC Listings](#hpc-listings) is for: * Sharing of HPC-relevant, short-format, lessons that use the teaching approach and lesson design from The Carpentries. * Collaborative lesson development and peer review. @@ -96,9 +96,9 @@ licensed [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/)).
    -## HPC Carpentry Listings +## HPC Listings -The HPC Carpentry Listings are a place for people to share HPC-oriented, +The HPC Listings are a place for people to share HPC-relevant, short-format, lessons that use the teaching approach and lesson design from The Carpentries. @@ -107,7 +107,7 @@ If you are interested in having a lesson included in our listings, please -->