groupwork_base.py

# Prerequisite: have Chrome installed or change the line driver = webdriver.Chrome() to whatever you prefer
# e.g. driver = webdriver.Firefox(). This is a basic version, if this fails, the website has fundamentally changed.
# and you'll need to figure out where. # and you'll need to figure out where. Use time.sleep(5) to pause on relevant,
# pages to see content better, and inspect element to see how formatting has changed
#
# Don't be alarmed when Selenium opens Chrome, this is needed for fetching data, as the data is
# loaded dynamically based on the fragments, and cannot be parsed via a standard. By isn't needed
# but the old way with just normal Selenium is deprecated anyways, so better to use it. WebDriverWait
# and EC (expected conditions) are needed, as content is dynamically loaded, and otherwise failures
# may occur. If parsing using Selenium, remember to use expected conditions to wait for content to load before
# interacting with it. XPath selectors are used, you can search documentation on that.
# Final parse is done using Beautiful Soup, since it's a little faster, but if you're a future subcomm
# reading this, feel free to use Selenium again if you only want to learn one.

import sys
import time

from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import NoSuchElementException, TimeoutException

# A known test course, that can be used to quickly verify whether this is working as intended or not
KNOWN_TEST_COURSE = "COMP6080"

# Wait a max of 20 second before giving up on a necessary element of a page loading. Most of the time, it won't take
# that long
MAX_WAIT_TIME = 20
# Look every 1 seconds to see if it's stopped changing
POLL_FREQUENCY = 1
INVALID_COUNT = -1
standard_XPath = "//dd"

# Waits until the number of the element is consistent to show that dynamic content is fully loaded,
# In this case, description details tag is used, since the elements we're fetching have that tag,
# and it's also one of the most common tags on the page.
def wait_for_stabilisation(web_driver, XPath):
    end_time = time.time() + MAX_WAIT_TIME
    last_count = INVALID_COUNT
    while time.time() < end_time:
        elements = web_driver.find_elements(By.XPATH, XPath)
        count = len(elements)
        if count == last_count and count > 0:
            return
        last_count = count
        time.sleep(POLL_FREQUENCY)
    raise TimeoutException("Elements did not stabilize in time")

# Uses your version of Chrome to run the script, please have Chrome installed, or alternatively, rewrite this to
#use whatever you usually use
driver = webdriver.Chrome()  # pylint: disable=not-callable  # selenium false positive
driver.get("https://www.unsw.edu.au/course-outlines")
# Sets wait times
wait = WebDriverWait(driver, MAX_WAIT_TIME)

# Needs to accept cookies when working via Selenium unfortunately otherwise overlay will block access
# to form. Can also have information sent via Javascript execution probably, but no clue about UNSW's
# level of protection against external scripting.
try:
    cookie_button = driver.find_element(By.ID, "onetrust-accept-btn-handler")
    cookie_button.click()
except NoSuchElementException:
    print("Cookie banner not found, possibly already handled")

# This is the search form's Id and the submit button's id, wait til they both exist,
# fill in the form and send keys
search_form_id = "degree-search-input"
search_form_submit_button = "degree-search-submit"
try:
    wait.until(
        EC.element_to_be_clickable((By.ID, search_form_submit_button))
    )
    # Fills in the form
    wait.until(
        EC.presence_of_element_located((By.ID, search_form_id))
    ).send_keys(KNOWN_TEST_COURSE)
    wait.until(
        EC.element_to_be_clickable((By.ID, search_form_submit_button))
    ).click()
except TimeoutException:
    # Content not loaded in time
    print(f"Element not found on webpage when searching from homepage: {KNOWN_TEST_COURSE}")
    sys.exit(1)

# As content is dynamically loaded, wait until it is loaded to try getting the link
try:
    wait = WebDriverWait(driver, MAX_WAIT_TIME)
    wait.until(
        EC.element_to_be_clickable((By.XPATH, f"//a/span[normalize-space(text())='{KNOWN_TEST_COURSE}']"))
    ).click()
except TimeoutException:
    # Content not loaded in time
    print(f"Element not found on webpage when trying to fetch link {KNOWN_TEST_COURSE}")
    sys.exit(1)

# Note: We can cache the pages generated by this program to avoid parsing again, and just use the
# direct link, however I recommend just reloading every time, since links will change year to year.
try:
    wait_for_stabilisation(driver, standard_XPath)
except TimeoutException:
    # Content not loaded in time
    print(f"Element not found on webpage when trying to fetch link {KNOWN_TEST_COURSE}")
    sys.exit(1)
html = driver.page_source

# Puts the HTML in nicer beautiful soup format for the final parse, it's quicker than using Selenium
soup = BeautifulSoup(html, 'html.parser')

# Gets whether it's group work or not, all group work specifications is stored under a tag called description details,
# and it conveniently enough can be parsed out. There's a bunch of unrelated description details tags, so
# we get all the ones that contain the exact text "Group", since that's a standard way of formatting
# group projects or tasks worth a percentage of the mark for the course. The below is approximately the
# format from the webpage:
# <dt>Assessment Format: <dt> <dd>Group<dd>
potential_group_tags = soup.find_all('dd')
groupwork = False
for potential_group_tag in potential_group_tags:
    if potential_group_tag.get_text() == "Group":
        groupwork = True
        break
if groupwork:
    print(f"Groupwork found in {KNOWN_TEST_COURSE}!")
else:
    print(f"Groupwork not found in {KNOWN_TEST_COURSE}!")

driver.quit()