Build Data #14
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Data | |
| on: | |
| schedule: | |
| - cron: "0 12 * * 4" | |
| workflow_dispatch: | |
| jobs: | |
| Build-Data: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Get current date | |
| id: date | |
| run: echo "::set-output name=date::$(date +'%Y-%m-%d')" | |
| - name: checkout repo content | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| ref: ${{ github.event.pull_request.head.ref }} # checkout the repository content | |
| - name: setup python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.14.4" # install the python version needed | |
| - name: Install poetry | |
| uses: abatilo/actions-poetry@v4 | |
| - name: Setup a local virtual environment (if no poetry.toml file) | |
| run: | | |
| poetry config virtualenvs.create true --local | |
| poetry config virtualenvs.in-project true --local | |
| - uses: actions/cache@v3 | |
| name: Define a cache for the virtual environment based on the dependencies lock file | |
| with: | |
| path: ./.venv | |
| key: venv-${{ hashFiles('poetry.lock') }} | |
| - name: Install the project dependencies | |
| run: poetry install | |
| - name: run scrapers | |
| run: | | |
| cd src/toronto_open_events | |
| poetry run run_all_scrapers | |
| - name: transform data | |
| run: | | |
| cd src/toronto_open_events | |
| poetry run dedupe_scraped_data | |
| poetry run create_xcal_feeds | |
| - uses: EndBug/add-and-commit@v9 # You can change this to use a specific version. | |
| with: | |
| add: "." | |
| # The name of the user that will be displayed as the author of the commit. | |
| # Default: depends on the default_author input | |
| author_name: Github Action Robot | |
| # The email of the user that will be displayed as the author of the commit. | |
| # Default: depends on the default_author input | |
| author_email: dontclick@dontclick.com | |
| # The message for the commit. | |
| # Default: 'Commit from GitHub Actions (name of the workflow)' | |
| message: "New data for ${{ steps.date.outputs.date }}" | |
| new_branch: newest-data | |
| # Whether to push the commit and, if any, its tags to the repo. It can also be used to set the git push arguments (see the paragraph below for more info) | |
| # Default: true | |
| push: origin newest-data --set-upstream --force |