Merge pull request #59 from tidy-finance/align-osap-beginning-of-month-and-scale-returns

Align OSAP download with beginning-of-month and scale returns

Author: christophscheuch

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 ## Unreleased
 
+- **OSAP download aligned with beginning-of-month and scaled returns:**
+  `download_data("Open Source Asset Pricing")` now aligns the `date`
+  column to the beginning of the month (the dataset previously returned
+  end-of-month dates), matching the convention used by the other
+  download functions. All predictor columns are monthly long-short
+  returns expressed in percent and are now divided by 100 to return
+  plain numeric (decimal) returns.
 - **`sorting_variable` is now optional for `factor_library`:** Calling
   `download_data("Tidy Finance", "factor_library")` without a
   `sorting_variable` now returns the default portfolio construction for

pyproject.toml

@@ -8,7 +8,7 @@ authors = [{name = "Christoph Frey", email = "christoph.frey@gmail.com"},
 	   {name = "Christoph Scheuch", email = "christoph@tidy-intelligence.com"},
        {name = "Stefan Voigt", email = "stefan.voigt@econ.ku.dk"}
            ]
-version = "0.3.1.dev1"
+version = "0.3.1.dev2"
 description = "Tidy Finance Helper Functions"
 readme = "README.md"
 license = "MIT"

tests/test_download_data_osap.py

@@ -19,7 +19,7 @@ def test_downloads_and_processes_all_rows():
     """Test downloads and processes all rows."""
     raw = pd.DataFrame(
         {
-            "date": ["2020-01-01", "2020-02-01"],
+            "date": ["2020-01-31", "2020-02-29"],
             "LongName": [1, 2],
         }
     )
@@ -28,17 +28,20 @@ def test_downloads_and_processes_all_rows():
 
     assert isinstance(result, pd.DataFrame)
     assert list(result.columns) == ["date", "long_name"]
+    # Dates are aligned to the beginning of the month.
     assert list(result["date"]) == [
         pd.Timestamp("2020-01-01"),
         pd.Timestamp("2020-02-01"),
     ]
+    # Percentage returns are scaled to numeric (decimal) values.
+    assert list(result["long_name"]) == [0.01, 0.02]
 
 
 def test_filters_rows_when_both_dates_are_supplied():
     """Test filters rows when both dates are supplied."""
     raw = pd.DataFrame(
         {
-            "date": ["2020-01-01", "2020-02-01", "2020-03-01"],
+            "date": ["2020-01-31", "2020-02-29", "2020-03-31"],
             "value": [1, 2, 3],
         }
     )
@@ -49,7 +52,7 @@ def test_filters_rows_when_both_dates_are_supplied():
 
     assert len(result) == 1
     assert result["date"].iloc[0] == pd.Timestamp("2020-02-01")
-    assert result["value"].iloc[0] == 2
+    assert result["value"].iloc[0] == 0.02
 
 
 def test_returns_empty_dataframe_after_download_failure():

tidyfinance/download_open_source.py

@@ -1099,8 +1099,14 @@ def _download_data_osap(
     Downloads the data from the Open Source Asset Pricing project at
     https://www.openassetpricing.com/data/ from Google Sheets using a
     specified sheet ID, processes the data by converting column names
-    to snake_case, and optionally filters the data based on a provided
-    date range.
+    to snake_case, aligning the date to the beginning of the month,
+    scaling the percentage long-short returns to numeric values, and
+    optionally filters the data based on a provided date range.
+
+    The dataset contains monthly long-short returns of the predictor
+    portfolios. Every column other than ``date`` is a return expressed
+    in percent, so all of them are divided by 100 to convert them into
+    plain numeric (decimal) returns.
 
     Parameters
     ----------
@@ -1121,9 +1127,11 @@ def _download_data_osap(
     -------
     pd.DataFrame
         A data frame containing the processed data. The column names
-        are converted to snake_case, and the data is filtered by the
-        specified date range if 'start_date' and 'end_date' are
-        provided.
+        are converted to snake_case, the ``date`` column is aligned to
+        the beginning of the month, all predictor columns (long-short
+        returns in percent) are divided by 100 to obtain plain numeric
+        (decimal) returns, and the data is filtered by the specified
+        date range if 'start_date' and 'end_date' are provided.
 
     Examples
     --------
@@ -1158,12 +1166,21 @@ def _download_data_osap(
         return raw_data
 
     if "date" in raw_data.columns:
-        raw_data["date"] = pd.to_datetime(raw_data["date"], errors="coerce")
+        raw_data["date"] = (
+            pd.to_datetime(raw_data["date"], errors="coerce")
+            .dt.to_period("M")
+            .dt.start_time
+        )
 
     raw_data.columns = [
         _transfrom_to_snake_case(col) for col in raw_data.columns
     ]
 
+    # All columns except the date are long-short returns in percent, so
+    # scale them to plain numeric (decimal) returns.
+    return_columns = [col for col in raw_data.columns if col != "date"]
+    raw_data[return_columns] = raw_data[return_columns] / 100
+
     if start_date and end_date:
         raw_data = raw_data.query("@start_date <= date <= @end_date")