Export BugSigDB #27968
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| on: | |
| schedule: | |
| - cron: "55 * * * *" # UTC Time | |
| workflow_dispatch: | |
| name: Export BugSigDB | |
| jobs: | |
| export-bugsigdb: | |
| runs-on: ubuntu-latest | |
| container: bioconductor/bioconductor_docker:devel | |
| steps: | |
| - name: Checkout BugSigDBExports | |
| uses: actions/checkout@v7 | |
| with: | |
| path: BugSigDBExports | |
| fetch-depth: 1 | |
| - name: Install BugSigDBExports dependencies | |
| run: | | |
| pkgs <- c('rvest', 'readr', 'plyr', 'BiocFileCache', 'R.utils', 'testthat', 'stringr', 'lubridate', 'dplyr') | |
| BiocManager::install(pkgs) | |
| shell: Rscript {0} | |
| - name: Install bugsigdbr | |
| run: | | |
| Rscript -e "BiocManager::install('bugsigdbr')" | |
| - name: Setup git config | |
| run: | | |
| cd $GITHUB_WORKSPACE/BugSigDBExports | |
| git config user.name "GitHub Actions Bot" | |
| git config user.email "actions@github.com" | |
| - name: Export BugSigDB | |
| id: export | |
| run: | | |
| BUGSIGDB_TIMESTAMP="$(date -u +%Y-%m-%d_%H:%M_UTC)" | |
| echo "BUGSIGDB_TIMESTAMP=$BUGSIGDB_TIMESTAMP" >> $GITHUB_ENV | |
| echo $BUGSIGDB_TIMESTAMP | |
| Rscript $GITHUB_WORKSPACE/BugSigDBExports/inst/scripts/dump_release.R $BUGSIGDB_TIMESTAMP $GITHUB_WORKSPACE/BugSigDBExports false | |
| mv *csv $GITHUB_WORKSPACE/BugSigDBExports | |
| timeout-minutes: 15 | |
| - name: Track file growth | |
| run: | | |
| library(R.utils) | |
| github_workspace <- Sys.getenv("GITHUB_WORKSPACE") | |
| setwd(file.path(github_workspace, "BugSigDBExports")) | |
| bugsigdb_timestamp <- Sys.getenv("BUGSIGDB_TIMESTAMP") | |
| number_of_lines <- c("timestamp" = bugsigdb_timestamp) | |
| files <- sort(list.files(pattern=".(csv|gmt)")) | |
| files <- files[files != "file_size.csv"] | |
| for (a_file in files) { | |
| number_of_lines[a_file] <- countLines(a_file)[1] | |
| } | |
| write.table(as.matrix(t(number_of_lines)), | |
| "file_size.csv", | |
| append = TRUE, | |
| col.names = FALSE, | |
| row.names = FALSE) | |
| # Keep only last 30 days | |
| df <- read.table("file_size.csv", header=TRUE) | |
| df$date <- as.Date(df[,1], format="%Y-%m-%d") | |
| df <- df[df$date >= Sys.Date() - 30, ] | |
| df$date <- NULL | |
| write.table(df, "file_size.csv", col.names=TRUE, row.names=FALSE) | |
| shell: Rscript {0} | |
| - name: Check for dramatic row count drop | |
| run: | | |
| Rscript -e " | |
| path <- file.path(Sys.getenv('GITHUB_WORKSPACE'), 'BugSigDBExports', 'file_size.csv') | |
| df <- read.table(path, header = TRUE) | |
| print(head(df, 2)) | |
| full_dump_col <- which(colnames(df) == 'full_dump.csv') | |
| print(paste('full_dump_col:', full_dump_col)) | |
| print(paste('nrow:', nrow(df))) | |
| if (length(full_dump_col) > 0 && nrow(df) > 1) { | |
| current <- as.numeric(df[nrow(df), full_dump_col]) | |
| previous <- as.numeric(df[nrow(df)-1, full_dump_col]) | |
| drop_pct <- (previous - current) / previous * 100 | |
| print(paste('drop_pct:', drop_pct)) | |
| if (drop_pct > 10) { | |
| cat(sprintf('DROP_DETECTED=true\nDROP_PCT=%.1f\n', drop_pct), | |
| file = Sys.getenv('GITHUB_ENV'), append = TRUE) | |
| } | |
| } | |
| " | |
| - name: Open issue on dramatic drop | |
| if: ${{ env.DROP_DETECTED == 'true' }} | |
| run: | | |
| curl -X POST \ | |
| -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ | |
| -H "Accept: application/vnd.github.v3+json" \ | |
| https://api.github.com/repos/${{ github.repository }}/issues \ | |
| -d "{ | |
| \"title\": \"Warning: dramatic row count drop detected\", | |
| \"body\": \"The full_dump.csv row count dropped by ${{ env.DROP_PCT }}% in the latest export. This may indicate a network issue during download. Please check the [latest run](https://github.com/${{ github.repository }}/actions).\" | |
| }" | |
| - name: Commit Exports | |
| id: commit_exports | |
| run: | | |
| cd $GITHUB_WORKSPACE/BugSigDBExports | |
| for export in `find . -type f -iname \*.gmt -o -iname \*.csv` | |
| do | |
| export=$(echo $export | sed 's/\.\///g') | |
| git_diff=$(git diff --numstat $export | sed -e 's/\t//g') | |
| if [ "$git_diff" = "11$export" -o "$git_diff" = "" ]; then | |
| echo "No changes to commit for $export" | |
| else | |
| echo "Add changes for $export" | |
| git add $export | |
| fi | |
| done | |
| echo "Check if any file contents have added to the index" | |
| has_updates=$(git diff --cached --shortstat) | |
| if [ -n "$has_updates" ]; then | |
| git add file_size.csv | |
| git commit -m "Hourly export update" | |
| git push -u origin HEAD | |
| echo "pushed=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "No content changes to commit" | |
| echo "pushed=false" >> $GITHUB_OUTPUT | |
| fi |