"""Loaders for standard public (non-vintage) sources.
tidyfinance (MIT) is the primary backend: Fama-French, Goyal-Welch, FRED, JKP, q-factors, OSAP.
Narrow direct-source adapters cover files whose exact author bytes matter for a reproduction:
the U.S. daily momentum factor, the frozen Global-q HXZ release, the paired Pastor--Stambaugh
liquidity releases, the frozen HKM paper archive, and caller-supplied AQR and Daniel--Moskowitz
carriers. Downloading adapters use memory only; path-only adapters never download or persist source
bytes. All record content digests.
These loaders return **tidy frames** (the package convention — see
:mod:`numeraire_dataset.builders` for the vintage-aware FRED-MD builder that tidyfinance does not
cover). Strict source frames attach inspectable provenance; view helpers return their
``data_vintage`` explicitly. :func:`to_timeseries_view` and :func:`to_multiasset_view` optionally
turn frames into numeraire point-in-time views; they import ``numeraire`` lazily, so the frame
loaders stay numeraire-independent.
For an exact reference reproduction, pin a fixture instead of a live pull — tidyfinance fetches the
current release, which drifts, and the live predictor sets differ slightly from any one paper's
(e.g. Goyal-Welch has no ``dfr``). Pin the ``tidyfinance`` version so a pull is reproducible.
"""
from __future__ import annotations
from importlib.metadata import version
from pathlib import Path
from typing import TYPE_CHECKING, Any, Literal, cast
import numpy as np
import pandas as pd
import tidyfinance as tf
from numeraire_dataset import (
_aqr_bab,
_daniel_moskowitz,
_global_q,
_hkm,
_ken_french,
_pastor_stambaugh,
)
from numeraire_dataset._compat import return_type_kwargs
if TYPE_CHECKING:
from numeraire.core.data import TimeSeriesView
_TF_VERSION = version("tidyfinance")
KEN_FRENCH_DAILY_MOMENTUM_URL = _ken_french.KEN_FRENCH_DAILY_MOMENTUM_URL
PASTOR_STAMBAUGH_LIQUIDITY_URL = _pastor_stambaugh.PASTOR_STAMBAUGH_LIQUIDITY_URL
PASTOR_STAMBAUGH_HISTORICAL_BETA_PORTFOLIOS_URL = (
_pastor_stambaugh.PASTOR_STAMBAUGH_HISTORICAL_BETA_PORTFOLIOS_URL
)
AQR_BAB_ORIGINAL_PAGE = _aqr_bab.AQR_BAB_ORIGINAL_PAGE
AQR_TERMS_URL = _aqr_bab.AQR_TERMS_URL
HXZ_Q4_FACTOR_COLUMNS = _global_q.HXZ_Q4_FACTOR_COLUMNS
DANIEL_MOSKOWITZ_DATA_PAGE = _daniel_moskowitz.DANIEL_MOSKOWITZ_DATA_PAGE
DANIEL_MOSKOWITZ_DOCUMENTATION_URL = _daniel_moskowitz.DANIEL_MOSKOWITZ_DOCUMENTATION_URL
DANIEL_MOSKOWITZ_ARCHIVE_FILENAME = _daniel_moskowitz.DANIEL_MOSKOWITZ_ARCHIVE_FILENAME
DANIEL_MOSKOWITZ_ARCHIVE_MEMBERS = _daniel_moskowitz.DANIEL_MOSKOWITZ_ARCHIVE_MEMBERS
DANIEL_MOSKOWITZ_DAILY_TOTAL_MEMBER = _daniel_moskowitz.DANIEL_MOSKOWITZ_DAILY_TOTAL_MEMBER
DANIEL_MOSKOWITZ_MONTHLY_TOTAL_MEMBER = _daniel_moskowitz.DANIEL_MOSKOWITZ_MONTHLY_TOTAL_MEMBER
DanielMoskowitzMemberContract = _daniel_moskowitz.DanielMoskowitzMemberContract
DanielMoskowitzArchiveContract = _daniel_moskowitz.DanielMoskowitzArchiveContract
DanielMoskowitzMomentumData = _daniel_moskowitz.DanielMoskowitzMomentumData
HKM_DATA_PAGE = _hkm.HKM_DATA_PAGE
HKM_ARCHIVE_URL = _hkm.HKM_ARCHIVE_URL
HKM_ARCHIVE_SHA256 = _hkm.HKM_ARCHIVE_SHA256
HKM_ARCHIVE_MEMBERS = _hkm.HKM_ARCHIVE_MEMBERS
HKM_ASSET_CLASSES = _hkm.HKM_ASSET_CLASSES
HKMFrequency = _hkm.HKMFrequency
HKMAssetClass = _hkm.HKMAssetClass
HKMPaperData = _hkm.HKMPaperData
HKMAssetClassData = _hkm.HKMAssetClassData
_PROVENANCE_ATTR = "numeraire_dataset:provenance"
FFPortfolioSet = Literal["industry_10", "size_bm_25", "momentum"]
FFPortfolioFrequency = Literal["monthly", "daily"]
FFFactorFrequency = Literal["monthly", "daily"]
_FF_PORTFOLIO_DATASETS: dict[tuple[str, str], str] = {
("industry_10", "monthly"): "10 Industry Portfolios",
("industry_10", "daily"): "10 Industry Portfolios [Daily]",
("size_bm_25", "monthly"): "25 Portfolios Formed on Size and Book-to-Market (5 x 5)",
("size_bm_25", "daily"): "25 Portfolios Formed on Size and Book-to-Market (5 x 5) [Daily]",
("momentum", "monthly"): "Momentum Factor (Mom)",
("momentum", "daily"): "Momentum Factor (Mom) [Daily]",
}
_FF_FRAME_RECIPES = {
"monthly": "availability-month-end-preserve-missing-v1",
"daily": "source-date-preserve-missing-v1",
}
_FF_FACTOR_RECIPES = {
"monthly": "availability-month-end-complete-v1",
"daily": "source-date-complete-v1",
}
_FF_FACTOR_DATASETS = {
"monthly": "Fama/French 3 Factors",
"daily": "Fama/French 3 Factors [Daily]",
}
_FF_FACTOR_COLUMNS = ("mkt_excess", "smb", "hml", "risk_free")
_FF_VIEW_RECIPES = {
"monthly": "availability-month-end-complete-case-v1",
"daily": "source-date-complete-case-v1",
}
[docs]
def data_vintage(dataset: str) -> str:
"""Provenance stamp for a live tidyfinance pull (``tidyfinance:<dataset>@<version>``)."""
return f"tidyfinance:{dataset}@{_TF_VERSION}"
[docs]
def frame_provenance(frame: pd.DataFrame) -> dict[str, str]:
"""Return a copy of source provenance attached by a public frame loader.
Pandas preserves ``DataFrame.attrs`` through many operations but does not make that a universal
guarantee. Read the metadata before an unrelated transform if it must travel separately. The
package's own preparation, filtering, and sorting paths stamp provenance only after the final
frame has been assembled.
"""
value = frame.attrs.get(_PROVENANCE_ATTR)
if not isinstance(value, dict) or not all(
isinstance(key, str) and isinstance(item, str) for key, item in value.items()
):
raise ValueError("frame has no valid numeraire-dataset provenance")
return cast(dict[str, str], value).copy()
[docs]
def frame_data_vintage(frame: pd.DataFrame) -> str:
"""Return the non-secret ``data_vintage`` attached by a public frame loader."""
provenance = frame_provenance(frame)
try:
return provenance["data_vintage"]
except KeyError:
raise ValueError("frame provenance has no data_vintage") from None
def _stamp_provenance(frame: pd.DataFrame, provenance: dict[str, str]) -> pd.DataFrame:
if not provenance.get("data_vintage"):
raise ValueError("source provenance requires a data_vintage")
frame.attrs[_PROVENANCE_ATTR] = provenance.copy()
return frame
def _stamp_tidyfinance_portfolio(
frame: pd.DataFrame, *, dataset: str, freq: FFPortfolioFrequency
) -> pd.DataFrame:
recipe = _FF_FRAME_RECIPES[freq]
return _stamp_tidyfinance_frame(frame, dataset=dataset, recipe=recipe)
def _stamp_tidyfinance_frame(
frame: pd.DataFrame,
*,
dataset: str,
recipe: str,
extra_provenance: dict[str, str] | None = None,
) -> pd.DataFrame:
vintage = data_vintage(f"{dataset}/{recipe}")
provenance = {
"backend": "tidyfinance",
"dataset": dataset,
"tidyfinance_version": _TF_VERSION,
"recipe": recipe,
"data_vintage": vintage,
}
if extra_provenance is not None:
provenance.update(extra_provenance)
return _stamp_provenance(
frame,
provenance,
)
def _date_bound(value: str, *, name: str) -> pd.Timestamp:
try:
bound = pd.Timestamp(value)
except (TypeError, ValueError) as exc:
raise ValueError(f"{name} is not a valid date") from exc
if pd.isna(bound):
raise ValueError(f"{name} is not a valid date")
if bound.tz is not None:
raise ValueError(f"{name} must be timezone-naive")
return bound.normalize()
def _ff_factor_query_bounds(
*, start_date: str, end_date: str, freq: FFFactorFrequency
) -> tuple[pd.Timestamp, pd.Timestamp, str, str]:
start = _date_bound(start_date, name="start_date")
end = _date_bound(end_date, name="end_date")
if start > end:
raise ValueError("start_date must not be after end_date")
query_start = start
query_end = end
if freq == "monthly":
# The backend filters month-start source labels. Broaden to the boundary months, then apply
# the public bounds to canonical month-end availability timestamps after normalization.
query_start = start.to_period("M").start_time.normalize()
query_end = end.to_period("M").end_time.normalize()
return start, end, query_start.date().isoformat(), query_end.date().isoformat()
def _select_ff_factors(
frame: pd.DataFrame,
*,
start: pd.Timestamp,
end: pd.Timestamp,
query_start: str,
query_end: str,
dataset: str,
) -> tuple[pd.DataFrame, dict[str, str]]:
selected = frame.loc[frame["date"].between(start, end)].reset_index(drop=True)
if selected.empty:
raise ValueError(f"no {dataset!r} observations in the requested date range")
provenance = {
"requested_start_date": start.date().isoformat(),
"requested_end_date": end.date().isoformat(),
"backend_query_start_date": query_start,
"backend_query_end_date": query_end,
"selected_start_date": pd.Timestamp(selected["date"].iloc[0]).date().isoformat(),
"selected_end_date": pd.Timestamp(selected["date"].iloc[-1]).date().isoformat(),
"selected_rows": str(len(selected)),
}
return selected, provenance
[docs]
def load_ff_factors(
*,
freq: FFFactorFrequency = "monthly",
start_date: str = "1926-07-01",
end_date: str = "2023-12-31",
) -> pd.DataFrame:
"""Fama-French 3-factor tidy frame (``date, mkt_excess, smb, hml, risk_free``), in decimals.
``mkt_excess`` is the excess market return, ``risk_free`` the 1-month T-bill — the canonical
academic market/risk-free pair. Monthly source period labels are moved from month-start to
month-end, when the returns are realized. ``start_date`` and ``end_date`` are then applied
inclusively to those canonical timestamps (not to the backend's month-start labels). The strict
schema rejects missing or non-finite factor values and attaches inspectable requested/backend/
selected bounds plus tidyfinance version/recipe provenance; it does not pretend to have a
raw-byte digest that the backend does not expose.
"""
if freq not in _FF_FACTOR_RECIPES:
frequencies = ", ".join(sorted(_FF_FACTOR_RECIPES))
raise ValueError(f"unknown freq {freq!r}; choose one of: {frequencies}")
start, end, query_start, query_end = _ff_factor_query_bounds(
start_date=start_date,
end_date=end_date,
freq=freq,
)
dataset = _FF_FACTOR_DATASETS[freq]
downloaded = tf.download_data( # pyright: ignore[reportAttributeAccessIssue] # no stubs
domain="Fama-French",
dataset=dataset,
start_date=query_start,
end_date=query_end,
)
frame = _prepare_ff_factors(downloaded, dataset=dataset, freq=freq)
selected, selection_provenance = _select_ff_factors(
frame,
start=start,
end=end,
query_start=query_start,
query_end=query_end,
dataset=dataset,
)
return _stamp_tidyfinance_frame(
selected,
dataset=dataset,
recipe=_FF_FACTOR_RECIPES[freq],
extra_provenance=selection_provenance,
)
def _ff_portfolio_dataset(portfolio_set: str, freq: str) -> str:
try:
return _FF_PORTFOLIO_DATASETS[(portfolio_set, freq)]
except KeyError:
sets = ", ".join(sorted({key[0] for key in _FF_PORTFOLIO_DATASETS}))
frequencies = ", ".join(sorted(_FF_FRAME_RECIPES))
if portfolio_set not in {key[0] for key in _FF_PORTFOLIO_DATASETS}:
raise ValueError(
f"unknown portfolio_set {portfolio_set!r}; choose one of: {sets}"
) from None
if freq not in _FF_FRAME_RECIPES:
raise ValueError(f"unknown freq {freq!r}; choose one of: {frequencies}") from None
supported = ", ".join(
sorted(
key_freq for key_set, key_freq in _FF_PORTFOLIO_DATASETS if key_set == portfolio_set
)
)
raise ValueError(
f"portfolio_set {portfolio_set!r} does not support freq {freq!r}; "
f"supported: {supported}"
) from None
def _prepare_ff_portfolios(
downloaded: object, *, dataset: str, freq: FFPortfolioFrequency
) -> pd.DataFrame:
if not isinstance(downloaded, pd.DataFrame):
raise TypeError(f"tidyfinance returned {type(downloaded).__name__}, not a DataFrame")
if downloaded.empty:
raise ValueError(f"tidyfinance returned no observations for {dataset!r}")
if not downloaded.columns.is_unique:
raise ValueError(f"tidyfinance dataset {dataset!r} has duplicate columns")
if "date" not in downloaded.columns:
raise ValueError(f"tidyfinance dataset {dataset!r} has no date column")
frame = downloaded.copy()
dates = pd.to_datetime(frame.pop("date"), errors="coerce")
if dates.isna().any():
raise ValueError(f"tidyfinance dataset {dataset!r} has invalid dates")
if freq == "monthly":
# French monthly observations are labelled at the start of their return month. A return is
# only realized at month-end, so expose that availability timestamp to numeraire.
dates = dates.dt.to_period("M").dt.to_timestamp("M")
date_index = pd.DatetimeIndex(dates)
if date_index.has_duplicates:
raise ValueError(f"tidyfinance dataset {dataset!r} has duplicate observation dates")
if not len(frame.columns):
raise ValueError(f"tidyfinance dataset {dataset!r} has no return columns")
try:
values = frame.to_numpy(dtype=np.float64)
except (TypeError, ValueError) as exc:
raise TypeError(f"tidyfinance dataset {dataset!r} has non-numeric returns") from exc
if np.isinf(values).any():
raise ValueError(f"tidyfinance dataset {dataset!r} has infinite returns")
out = pd.DataFrame(values, columns=frame.columns)
out.insert(0, "date", date_index.to_numpy())
return out.sort_values("date").reset_index(drop=True)
def _prepare_ff_factors(
downloaded: object, *, dataset: str, freq: FFFactorFrequency
) -> pd.DataFrame:
frame = _prepare_ff_portfolios(downloaded, dataset=dataset, freq=freq)
value_columns = [column for column in frame.columns if column != "date"]
if len(value_columns) != len(_FF_FACTOR_COLUMNS) or set(value_columns) != set(
_FF_FACTOR_COLUMNS
):
raise ValueError(
f"tidyfinance dataset {dataset!r} must contain exactly {_FF_FACTOR_COLUMNS!r}"
)
canonical = frame.loc[:, ["date", *_FF_FACTOR_COLUMNS]].copy()
if canonical.loc[:, list(_FF_FACTOR_COLUMNS)].isna().any().any():
raise ValueError(f"tidyfinance dataset {dataset!r} has missing factor values")
return canonical
def _canonicalize_momentum(frame: pd.DataFrame, *, dataset: str) -> pd.DataFrame:
columns = [column for column in frame.columns if column != "date"]
if len(columns) != 1:
raise ValueError(f"{dataset!r} must contain exactly one momentum return column")
source_column = columns[0]
if str(source_column).strip().casefold() not in {"mom", "wml"}:
raise ValueError(f"{dataset!r} has unexpected momentum column {source_column!r}")
return frame.rename(columns={source_column: "mom"})
[docs]
def load_ff_momentum(
*,
freq: FFPortfolioFrequency = "monthly",
start_date: str = "1926-07-01",
end_date: str = "2023-12-31",
timeout: float = 30.0,
) -> pd.DataFrame:
"""Ken French U.S. momentum factor as ``date, mom`` decimal returns.
Monthly data use the primary tidyfinance backend. Daily data fill a documented tidyfinance
coverage gap by downloading the official Ken French Data Library CSV zip over HTTPS. The daily
parser requires the published header and copyright footer, preserves trading-day timestamps,
converts percentages to decimals, and rejects duplicate dates, missing sentinels, and non-finite
values. ``start_date`` and ``end_date`` are inclusive.
Both paths attach inspectable metadata readable through :func:`frame_provenance` and
:func:`frame_data_vintage`. Daily provenance includes the exact download URL, the source's HTTP
``Last-Modified`` timestamp (or ``unreported``), the zip's SHA-256 digest, and the parsing
recipe. No downloaded file is persisted by this loader.
"""
dataset = _ff_portfolio_dataset("momentum", freq)
if freq == "daily":
frame, provenance = _ken_french.load_daily_momentum(
start_date=start_date,
end_date=end_date,
timeout=timeout,
)
return _stamp_provenance(frame, provenance)
downloaded = tf.download_data( # pyright: ignore[reportAttributeAccessIssue] # no stubs
domain="Fama-French", dataset=dataset, start_date=start_date, end_date=end_date
)
frame = _prepare_ff_portfolios(downloaded, dataset=dataset, freq=freq)
canonical = _canonicalize_momentum(frame, dataset=dataset)
return _stamp_tidyfinance_portfolio(canonical, dataset=dataset, freq=freq)
[docs]
def load_ff_portfolios(
*,
portfolio_set: FFPortfolioSet = "industry_10",
freq: FFPortfolioFrequency = "monthly",
start_date: str = "1926-07-01",
end_date: str = "2023-12-31",
) -> pd.DataFrame:
"""A public Ken French portfolio frame in decimal returns.
``portfolio_set`` selects the ten industry portfolios, the 25 Size--Book-to-Market portfolios,
or the momentum (Mom/WML) factor. All three support monthly and daily releases. Momentum is a
factor rather than a portfolio; this compatibility surface delegates to the more precisely
named :func:`load_ff_momentum`. The returned frame is wide and tidy: one ``date`` row per
observation and one column per investable return series. Monthly source labels are moved from
month-start to month-end because that is when the month's return is realized; daily dates are
preserved.
The loader validates source schema and dates and rejects infinite values, but preserves missing
returns. Use :func:`to_multiasset_view` (or :func:`load_ff_portfolio_view`) to apply
complete-case filtering and construct a returns-only numeraire view.
"""
dataset = _ff_portfolio_dataset(portfolio_set, freq)
if portfolio_set == "momentum":
return load_ff_momentum(
freq=freq,
start_date=start_date,
end_date=end_date,
)
downloaded = tf.download_data( # pyright: ignore[reportAttributeAccessIssue] # no stubs
domain="Fama-French", dataset=dataset, start_date=start_date, end_date=end_date
)
frame = _prepare_ff_portfolios(downloaded, dataset=dataset, freq=freq)
return _stamp_tidyfinance_portfolio(frame, dataset=dataset, freq=freq)
[docs]
def load_pastor_stambaugh_liquidity(
*,
start_date: str | None = None,
end_date: str | None = None,
url: str = PASTOR_STAMBAUGH_LIQUIDITY_URL,
timeout: float = 30.0,
) -> pd.DataFrame:
"""Official Pastor--Stambaugh aggregate-liquidity snapshot, with explicit units.
The returned monthly frame is timestamped at month-end and contains ``agg_liq`` and
``innov_liq`` exactly in the author's published numeric scale, ``liquidity_innovation`` as
``innov_liq / 100`` for the paper's equations 14--18 and numeraire-zoo, and ``traded_liq`` as a
decimal 10-minus-1 portfolio return. Only the author's documented pre-1968 ``-99`` values in
``traded_liq`` become missing; every other sentinel, malformed row, duplicate, or calendar gap
is rejected. ``start_date`` and ``end_date`` are inclusive against those month-end timestamps.
The author revises the full historical file, so this is a latest/revised snapshot, not a
point-in-time vintage suitable for historical forecasting. Provenance records the pinned Booth
URL, an allowlist of HTTP metadata, retrieval time, exact response SHA-256, parser/unit recipe,
and requested/selected slice. The content hash, rather than ``Last-Modified``, identifies the
data vintage. No response bytes are persisted. ``url`` may point to a newer year-stamped file
only inside the same official Booth author-data directory; mirrors are never substituted.
"""
frame, provenance = _pastor_stambaugh.load_liquidity(
start_date=start_date,
end_date=end_date,
url=url,
timeout=timeout,
)
return _stamp_provenance(frame, provenance)
[docs]
def load_pastor_stambaugh_historical_beta_portfolios(
*,
start_date: str | None = None,
end_date: str | None = None,
url: str = PASTOR_STAMBAUGH_HISTORICAL_BETA_PORTFOLIOS_URL,
timeout: float = 30.0,
) -> pd.DataFrame:
"""Official historical-liquidity-beta decile returns and traded 10-minus-1 factor.
Columns ``decile_1`` through ``decile_10`` are value-weighted monthly **total returns** in
decimal units; subtract the same-month Fama--French ``risk_free`` rate before passing them as
equation-14 test-asset excess returns. ``traded_liq`` is the author's supplied decimal 10-1
return (RF cancels) and is retained rather than recomputed. The source has no date field, so the
strict parser constructs month-end timestamps from its declared range, requires exactly one row
per declared month, and validates ``10-1 = decile_10 - decile_1`` within eighth-decimal
rounding. Inclusive slicing is applied only after the complete source envelope has passed
validation.
These are the author's historical-beta portfolios, beginning in 1968. They are not the
predicted-beta portfolios used for the paper's Table 6 and must not be presented as a Table 6
reproduction. The revision-prone latest snapshot carries the same URL/HTTP/SHA/recipe/slice
provenance contract as :func:`load_pastor_stambaugh_liquidity`; no raw bytes are persisted.
"""
frame, provenance = _pastor_stambaugh.load_historical_beta_portfolios(
start_date=start_date,
end_date=end_date,
url=url,
timeout=timeout,
)
return _stamp_provenance(frame, provenance)
[docs]
def load_aqr_bab_original(
path: str | Path,
*,
series: str = "U.S. Equities",
start_date: str | None = None,
end_date: str | None = None,
) -> pd.DataFrame:
"""Load a caller-supplied AQR BAB Original workbook as decimal monthly returns.
This is deliberately a **path-only runtime adapter**. It never downloads, searches for, copies,
or caches AQR bytes. The workbook must have the exact Original-paper sheet/header envelope and
contain the static 1929-04--2012-03 U.S. series. Office zip limits, active content, formulas,
dates, continuity, percent display format, and values are validated before the requested series
is returned as ``date, bab`` decimal returns. The workbook's underlying values are already
decimals; its ``0.00%`` cell format is presentation metadata, so values are not divided again.
Provenance, readable through :func:`frame_provenance`, records only the official AQR source and
terms pages, source-content SHA-256, parser/unit recipe, release semantics, selected slice, and
``redistributable=false``. It never contains the caller's local path. Install the optional
``aqr`` extra for the spreadsheet parser. AQR source data remain subject to AQR's terms and must
not be placed in a wheel, source tree, or redistributable fixture.
"""
frame, provenance = _aqr_bab.load_original(
path,
series=series,
start_date=start_date,
end_date=end_date,
)
return _stamp_provenance(frame, provenance)
[docs]
def load_daniel_moskowitz_momentum(
path: str | Path,
*,
contract: DanielMoskowitzArchiveContract,
) -> DanielMoskowitzMomentumData:
"""Load caller-supplied Daniel--Moskowitz daily/monthly momentum deciles.
This is a **path-only, never-download** adapter for ``DM_data_2014_02.tar.gz``. It reads the
compressed file once, validates every member in memory, and never extracts, copies, caches, or
persists archive bytes. The tar envelope must contain exactly the twelve root-level regular
text files documented by the authors; unexpected paths, links, devices, duplicate names, and
unsafe size/compression or text-record/field envelopes fail closed.
Secure traversal of every source-path component requires POSIX ``dir_fd`` and ``O_NOFOLLOW``
support; platforms without those primitives fail closed before reading the archive.
The authors document member names and portfolio semantics but not a machine-readable
delimiter, header, or column layout. A frozen :class:`DanielMoskowitzArchiveContract` is
therefore required: callers explicitly name date/decile columns, date formats, units,
data-record bounds, and any expected SHA/date/row identity envelope. No paper-exact columns are
guessed. Both returned frames have exactly ``date, decile_1, ..., decile_10, wml``; returns are
decimal simple returns
and ``wml`` is the documented long-decile-10 minus short-decile-1 arithmetic. Monthly dates are
normalized to month-end, monthly continuity is required, daily trading dates remain untouched,
and the paired members must cover exactly the same months.
:func:`frame_provenance` on either frame records the actual archive SHA-256, selected and paired
member names/SHA-256 values, an all-member manifest digest, parser-contract hash, unit recipe,
date/row envelope, and ``redistributable=false`` without retaining the local path. Unless the
caller supplies expected archive/member hashes, provenance labels identity as recorded but not
fully caller-pinned; the loader never establishes a paper-exact claim by itself.
The archive supplies momentum portfolios only. It does **not** supply the separate monthly
market total return, daily market excess return, or risk-free series needed to build the paper's
panic-state and conditional-moment inputs. Those must come from separately versioned sources.
"""
parsed, daily_provenance, monthly_provenance = _daniel_moskowitz.load_archive(
path,
contract=contract,
)
return DanielMoskowitzMomentumData(
daily=_stamp_provenance(parsed.daily, daily_provenance),
monthly=_stamp_provenance(parsed.monthly, monthly_provenance),
)
[docs]
def load_hkm_paper_data(
*,
frequency: HKMFrequency = "quarterly",
path: str | Path | None = None,
timeout: float = 30.0,
) -> HKMPaperData:
"""Load the fixed He--Kelly--Manela paper factors and 124 test assets.
With ``path=None``, the one official author ZIP is downloaded into memory. Passing ``path``
instead reads a caller-supplied local ``.zip`` through a POSIX ``dir_fd``/``O_NOFOLLOW``
traversal that fails closed on unsupported platforms. Both routes require the frozen official
archive SHA-256 before parsing. No ZIP, CSV, README, or bundled Julia bytes are extracted,
copied, cached, executed, or persisted.
Only the original-paper quarterly or monthly test-asset file is exposed. The archive's updated
daily/monthly/quarterly factor-only files are schema/calendar validated but deliberately have no
public selection route, preventing their 2013--2018 extension from being mixed into the paper's
1970Q1--2012Q4 cross-sectional sample.
``factors`` uses decimal units and canonical ``mkt_excess`` / ``risk_free`` labels.
``excess_returns`` contains the 124 distinct official assets after subtracting same-period RF;
its cross-class missingness is preserved. The redundant source ``All_01``--``All_124`` block
must exactly duplicate those assets cell-for-cell and is then omitted. ``asset_metadata`` maps
every asset to its class, within-class position, and duplicate ``All`` source column.
:meth:`HKMPaperData.complete_case` performs complete-case selection within one asset class,
never across the unbalanced 124-asset panel.
All three frames carry path-free :func:`frame_provenance` with archive/member hashes, recipe,
rows, units, source mode, and ``redistributable=false``. The author's page permits free
non-commercial use and provides the files as-is, but states no general redistribution grant;
the test assets also retain their original third-party source obligations.
"""
parsed, provenance = _hkm.load_paper_data(
frequency=frequency,
path=path,
timeout=timeout,
)
return HKMPaperData(
frequency=parsed.frequency,
factors=_stamp_provenance(parsed.factors, {**provenance, "frame_role": "factors"}),
excess_returns=_stamp_provenance(
parsed.excess_returns,
{**provenance, "frame_role": "excess_returns"},
),
asset_metadata=_stamp_provenance(
parsed.asset_metadata,
{**provenance, "frame_role": "asset_metadata"},
),
)
[docs]
def load_hxz_q4_factors(
*,
start_date: str | None = None,
end_date: str | None = None,
timeout: float = 30.0,
) -> pd.DataFrame:
"""Official Hou--Xue--Zhang q4 factors as month-end decimal returns.
The returned columns are exactly ``date, risk_free, mkt_excess, me, ia, roe``. The official
file is now a q5 carrier, but this loader explicitly selects the original 2015 model's
:data:`HXZ_Q4_FACTOR_COLUMNS` and excludes ``R_EG``. It validates the complete 1967-01 through
2024-12 release before applying inclusive ``start_date`` / ``end_date`` bounds, and converts the
source's percent values to decimals.
The loader is frozen to the byte identity of Global-q's 2025-02-16 release. It downloads the
official HTTPS CSV once into memory, follows redirects only within ``global-q.org``, and writes
no cache. Provenance records the fixed source URL, release date, SHA-256, complete and selected
row counts, unit conversion, q5 carrier, q4 selection, and explicit ``R_EG`` exclusion. The
source page states no redistribution licence, so only loader code—not source bytes—is shipped.
"""
frame, provenance = _global_q.load_q4_factors(
start_date=start_date,
end_date=end_date,
timeout=timeout,
)
return _stamp_provenance(frame, provenance)
[docs]
def load_hxz_q_factor_legs(
*,
start_date: str | None = None,
end_date: str | None = None,
timeout: float = 30.0,
) -> pd.DataFrame:
"""Official HXZ 2x3x3 size/investment/profitability portfolios in decimal returns.
Each month has all 18 ``rank_me`` x ``rank_ia`` x ``rank_roe`` cells, with positive ``nstocks``
and value-weighted total / ex-dividend returns in ``ret_vw`` / ``retx_vw``. The full 1967-01
through 2024-12 grid, unique keys, finite values, and percent units are validated before any
inclusive date slice is selected. Use ``ret_vw`` to rebuild the original q-factor legs.
Like :func:`load_hxz_q4_factors`, this is a memory-only adapter pinned to the official
2025-02-16 Global-q release. Inspect :func:`frame_provenance` for its URL, SHA-256, parser and
unit recipe, full/selected row counts, and non-redistributable-source marker.
"""
frame, provenance = _global_q.load_q_factor_legs(
start_date=start_date,
end_date=end_date,
timeout=timeout,
)
return _stamp_provenance(frame, provenance)
[docs]
def load_goyal_welch(
*, freq: str = "monthly", start_date: str = "1926-07-01", end_date: str = "2023-12-31"
) -> pd.DataFrame:
"""Goyal-Welch equity-premium frame: ``date, rp_div`` (excess market) + macro predictors."""
dataset = f"macro_predictors_{freq}"
return tf.download_data( # pyright: ignore[reportAttributeAccessIssue] # no stubs
domain="macro_predictors", dataset=dataset, start_date=start_date, end_date=end_date
)
[docs]
def to_timeseries_view(
df: pd.DataFrame,
*,
ret_col: str,
feature_cols: list[str] | None = None,
date_col: str = "date",
asset: str = "mkt",
horizon: int = 1,
return_type: str = "simple",
) -> TimeSeriesView:
"""A tidy ``(date, ret, features…)`` frame → single-asset numeraire ``TimeSeriesView``.
``feature_cols=None`` uses every column except ``date_col`` and ``ret_col``. Rows with any NaN
in the selected columns are dropped and the index is sorted. Requires the ``numeraire`` extra
(imported lazily so the frame loaders above do not depend on numeraire).
``return_type`` declares the algebra of ``ret_col`` to numeraire (``"simple"`` by default, or
``"log"`` when the frame carries a ``source_log_return``-style column). It is forwarded only
when it differs from ``"simple"`` and requires ``numeraire >= 0.3``; on an older numeraire a
non-simple value raises rather than silently mixing log and simple algebra.
"""
from numeraire.core.data import TimeSeriesView # lazy: keep frame loaders numeraire-free
feats = feature_cols
if feats is None:
feats = [c for c in df.columns if c not in (date_col, ret_col)]
clean = df.set_index(date_col)[[ret_col, *feats]].dropna().sort_index()
idx = pd.DatetimeIndex(pd.to_datetime(clean.index), name="date")
returns = pd.DataFrame({asset: clean[ret_col].to_numpy()}, index=idx)
features = pd.DataFrame(clean[feats].to_numpy(), index=idx, columns=feats)
return TimeSeriesView(
returns, features, horizon=horizon, **return_type_kwargs(TimeSeriesView, return_type)
)
[docs]
def to_multiasset_view(
df: pd.DataFrame,
*,
asset_cols: list[str] | None = None,
date_col: str = "date",
horizon: int = 1,
return_type: str = "simple",
) -> TimeSeriesView:
"""A tidy wide ``(date, asset returns...)`` frame -> returns-only ``TimeSeriesView``.
``asset_cols=None`` uses every column except ``date_col``. Rows with a missing return in any
selected asset are dropped (the dense multi-asset view has a common calendar); infinite values,
invalid dates, duplicate dates, and an empty complete-case sample are rejected. Requires the
optional ``numeraire`` extra, imported lazily.
``return_type`` declares the algebra of the asset returns to numeraire (``"simple"`` by default,
or ``"log"``). It is forwarded only when non-simple and requires ``numeraire >= 0.3``; on an
older numeraire a non-simple value raises rather than silently mixing return algebra.
"""
from numeraire.core.data import TimeSeriesView # lazy: keep frame loaders numeraire-free
if date_col not in df.columns:
raise ValueError(f"frame has no {date_col!r} column")
if not df.columns.is_unique:
raise ValueError("frame has duplicate columns")
assets = asset_cols if asset_cols is not None else [c for c in df.columns if c != date_col]
if not assets:
raise ValueError("select at least one asset return column")
if len(assets) != len(set(assets)):
raise ValueError("asset_cols must be unique")
missing = sorted(set(assets) - set(df.columns))
if missing:
raise ValueError(f"frame is missing asset columns: {missing}")
dates = pd.to_datetime(df[date_col], errors="coerce")
if dates.isna().any():
raise ValueError("frame has invalid dates")
index = pd.DatetimeIndex(dates, name="date")
if index.tz is not None:
raise ValueError("frame dates must be timezone-naive")
if index.has_duplicates:
raise ValueError("frame has duplicate dates")
try:
values = df.loc[:, assets].to_numpy(dtype=np.float64)
except (TypeError, ValueError) as exc:
raise TypeError("asset returns must be numeric") from exc
if np.isinf(values).any():
raise ValueError("asset returns contain infinite values")
returns = pd.DataFrame(values, index=index, columns=assets).dropna().sort_index()
if returns.empty:
raise ValueError("no complete multi-asset observations remain")
return TimeSeriesView(
returns, horizon=horizon, **return_type_kwargs(TimeSeriesView, return_type)
)
[docs]
def load_gw_view(
*,
freq: str = "monthly",
start_date: str = "1926-07-01",
end_date: str = "2023-12-31",
predictors: list[str] | None = None,
horizon: int = 1,
) -> tuple[Any, str]:
"""Goyal-Welch view (excess ``rp_div`` + predictors) + ``data_vintage``, ready for VoC/1-A.
Convenience over :func:`load_goyal_welch` + :func:`to_timeseries_view`; needs the ``numeraire``
extra. Return type is the numeraire ``TimeSeriesView`` (typed ``Any`` to avoid a hard import).
"""
df = load_goyal_welch(freq=freq, start_date=start_date, end_date=end_date)
view = to_timeseries_view(df, ret_col="rp_div", feature_cols=predictors, horizon=horizon)
return view, data_vintage(f"macro_predictors_{freq}")
[docs]
def load_ff_portfolio_view(
*,
portfolio_set: FFPortfolioSet = "industry_10",
freq: FFPortfolioFrequency = "monthly",
start_date: str = "1926-07-01",
end_date: str = "2023-12-31",
asset_cols: list[str] | None = None,
horizon: int = 1,
) -> tuple[Any, str]:
"""Ken French multi-asset view plus a non-secret ``data_vintage`` provenance stamp.
This is the optional numeraire bridge over :func:`load_ff_portfolios`. Tidyfinance-backed
vintages record its dataset/version and the calendar-normalization recipe. The direct daily
momentum fallback instead records the official URL, HTTP source timestamp, content SHA-256,
and parse recipe. Live releases can drift; reports that require fixed numbers should pin the
returned digest or a public fixture.
"""
frame = load_ff_portfolios(
portfolio_set=portfolio_set,
freq=freq,
start_date=start_date,
end_date=end_date,
)
provenance = frame_provenance(frame)
source_vintage = frame_data_vintage(frame)
view_recipe = _FF_VIEW_RECIPES[freq]
if provenance["backend"] == "tidyfinance":
vintage = data_vintage(f"{provenance['dataset']}/{view_recipe}")
else:
vintage = f"{source_vintage}/view:{view_recipe}"
view = to_multiasset_view(frame, asset_cols=asset_cols, horizon=horizon)
return view, vintage