From c8863a0aa1bf961c06842ca2b06998afbe135f32 Mon Sep 17 00:00:00 2001 From: edavidaja Date: Fri, 10 Jul 2026 11:58:23 -0400 Subject: [PATCH] unpin fsspec --- pins/boards.py | 23 +++++++++++++---------- pins/tests/helpers.py | 8 +++----- pins/tests/test_boards.py | 17 +++++++++-------- pins/tests/test_compat.py | 21 +++++++++++---------- pins/tests/test_constructors.py | 11 ++++++----- pins/utils.py | 11 +++++++++++ pyproject.toml | 2 +- 7 files changed, 54 insertions(+), 39 deletions(-) diff --git a/pins/boards.py b/pins/boards.py index 893be522..d19b19d0 100644 --- a/pins/boards.py +++ b/pins/boards.py @@ -22,7 +22,7 @@ from .drivers import REQUIRES_SINGLE_FILE, default_title, load_data, load_file, save_data from .errors import PinsError, PinsVersionError from .meta import Meta, MetaFactory, MetaRaw -from .utils import ExtendMethodDoc, inform, warn_deprecated +from .utils import ExtendMethodDoc, fs_protocol, inform, warn_deprecated from .versions import VersionRaw, guess_version, version_setup _log = logging.getLogger(__name__) @@ -858,22 +858,23 @@ def board_deparse(board: BaseBoard): else: allow_pickle = "" - prot = board.fs.protocol + prot = fs_protocol(board.fs) + prots = {prot} if isinstance(prot, str) else set(prot) - if prot == "rsc": + if "rsc" in prots: url = board.fs.api.server_url return f"board_connect(server_url={repr(url)}{allow_pickle})" - elif prot in ["file", ("file", "local")]: + elif "file" in prots: return f"board_folder({repr(board.board)}{allow_pickle})" - elif set(prot) == {"s3", "s3a"}: + elif prots & {"s3", "s3a"}: return f"board_s3({repr(board.board)}{allow_pickle})" - elif prot == "abfs": + elif "abfs" in prots: return f"board_azure({repr(board.board)}{allow_pickle})" - elif set(prot) == {"gcs", "gs"} or prot == "gs": + elif prots & {"gcs", "gs"}: return f"board_gcs({repr(board.board)}{allow_pickle})" - elif prot == "http": + elif "http" in prots: return f"board_url({repr(board.board)}, {board.pin_paths}{allow_pickle})" - elif prot == "dbc": + elif "dbc" in prots: return f"board_databricks({repr(board.board)}{allow_pickle})" else: raise NotImplementedError( @@ -930,7 +931,9 @@ def pin_meta(self, name, version=None): # a file. here we need to create a stripped down form of metadata, since # a metadata file does not exist (and we can't pull files from a version dir). path_to_pin = self.construct_path([pin_name]) - if self.fs.protocol == "http" and not path_to_pin.rstrip().endswith("/"): + prot = fs_protocol(self.fs) + is_http = prot == "http" or (not isinstance(prot, str) and "http" in prot) + if is_http and not path_to_pin.rstrip().endswith("/"): # create metadata, rather than read from a file return self.meta_factory.create_raw(path_to_pin, type="file", name=pin_name) diff --git a/pins/tests/helpers.py b/pins/tests/helpers.py index 5909f7de..ecf4c362 100644 --- a/pins/tests/helpers.py +++ b/pins/tests/helpers.py @@ -15,6 +15,7 @@ from pins.boards import BaseBoard, BoardRsConnect from pins.constructors import board_databricks +from pins.utils import fs_protocol DEFAULT_CREATION_DATE = datetime(2020, 1, 13, 23, 58, 59) @@ -65,10 +66,7 @@ def wrapper(*args, **kwargs): board = arg_value break - if board and ( - board.fs.protocol == "dbc" - or (hasattr(board.fs, "fs") and board.fs.fs.protocol == "dbc") - ): + if board and fs_protocol(board.fs) == "dbc": pytest.skip("All Databricks tests must be read only") return func(*args, **kwargs) @@ -104,7 +102,7 @@ def outer(f): # Assumes a fixture named board is passed to the test @wraps(f) def wrapper(board, *args, **kwargs): - if board.fs.protocol in names: + if fs_protocol(board.fs) in names: pytest.xfail() return f(board, *args, **kwargs) else: diff --git a/pins/tests/test_boards.py b/pins/tests/test_boards.py index 2d9741cc..ea24acda 100644 --- a/pins/tests/test_boards.py +++ b/pins/tests/test_boards.py @@ -16,6 +16,7 @@ from pins.errors import PinsError, PinsInsecureReadError, PinsVersionError from pins.meta import MetaRaw from pins.tests.helpers import DEFAULT_CREATION_DATE, rm_env, skip_if_dbc +from pins.utils import fs_protocol @fixture @@ -259,7 +260,7 @@ def test_board_pin_download_no_cache_error(board, tmp_path): assert meta.type == "file" # file boards work okay, since the board directory itself is the cache - if board.fs.protocol in ["file", ("file", "local")]: + if fs_protocol(board.fs) in ["file", ("file", "local")]: pytest.skip() # uncached boards should fail, since nowhere to store the download @@ -303,7 +304,7 @@ def test_board_pin_download_filename_multifile(board_with_cache, tmp_path): def test_board_pin_write_rsc_index_html(board, tmp_path: Path, snapshot): - if board.fs.protocol != "rsc": + if fs_protocol(board.fs) != "rsc": pytest.skip() df = pd.DataFrame({"x": [1, 2, None], "y": ["a", "b", "c"]}) @@ -451,7 +452,7 @@ def pin_name(): @pytest.fixture def pin_del(board, df, pin_name): # TODO: update when dbc boards no longer read-only - if board.fs.protocol == "dbc": + if fs_protocol(board.fs) == "dbc": pytest.skip() # 1min ago to avoid name collision one_min_ago = datetime.now() - timedelta(minutes=1) @@ -471,7 +472,7 @@ def pin_del(board, df, pin_name): @pytest.fixture def pin_prune(board, df, pin_name): # TODO: update when dbc boards no longer read-only - if board.fs.protocol == "dbc": + if fs_protocol(board.fs) == "dbc": pytest.skip() today = datetime.now() day_ago = today - timedelta(days=1, minutes=1) @@ -521,7 +522,7 @@ def test_board_pin_version_delete_older(board, pin_name, pin_del): def test_board_pin_version_delete_latest(board, pin_name, pin_del): meta_old, meta_new = pin_del - if board.fs.protocol == "rsc": + if fs_protocol(board.fs) == "rsc": with pytest.raises(PinsError) as exc_info: board.pin_version_delete(pin_name, meta_new.version.version) @@ -553,7 +554,7 @@ def test_board_pin_versions_prune_n(board, pin_prune, pin_name, n): @pytest.mark.parametrize("days", [1, 2]) def test_board_pin_versions_prune_days(board, pin_prune, pin_name, days): # Posit cannot handle days, since it involves pulling metadata - if board.fs.protocol == "rsc": + if fs_protocol(board.fs) == "rsc": with pytest.raises(NotImplementedError): board.pin_versions_prune(pin_name, days=days) return @@ -570,7 +571,7 @@ def test_board_pin_versions_prune_days(board, pin_prune, pin_name, days): def test_board_pin_versions_prune_days_protect_most_recent(board, pin_name): """To address https://github.com/rstudio/pins-python/issues/297""" # Posit cannot handle days, since it involves pulling metadata - if board.fs.protocol == "rsc": + if fs_protocol(board.fs) == "rsc": with pytest.raises(NotImplementedError): board.pin_versions_prune(pin_name, days=5) return @@ -612,7 +613,7 @@ def test_board_pin_versions_prune_days_protect_most_recent(board, pin_name): ) @skip_if_dbc def test_board_pin_search_name(board, df, search, matches): - if board.fs.protocol == "rsc": + if fs_protocol(board.fs) == "rsc": matches = ["derek/" + m for m in matches] # rsc doesn't search by title diff --git a/pins/tests/test_compat.py b/pins/tests/test_compat.py index 1477d1c3..03e68b6c 100644 --- a/pins/tests/test_compat.py +++ b/pins/tests/test_compat.py @@ -9,6 +9,7 @@ PATH_TO_MANIFEST_BOARD, ) from pins.tests.helpers import skip_if_dbc, xfail_fs +from pins.utils import fs_protocol NOT_A_PIN = "not_a_pin_abcdefg" PIN_CSV = "df_csv" @@ -19,7 +20,7 @@ @pytest.fixture(scope="session") def board(backend): board = backend.create_tmp_board(str(PATH_TO_EXAMPLE_BOARD.absolute())) - if board.fs.protocol == "dbc": + if fs_protocol(board.fs) == "dbc": board = backend.create_tmp_board(str(PATH_TO_EXAMPLE_BOARD_DBC)) yield board @@ -47,10 +48,10 @@ def test_compat_pin_list(board): src_sorted = sorted(board.pin_list()) dst_sorted = ["df_arrow", "df_csv", "df_rds", "df_unversioned"] - if board.fs.protocol == "rsc": + if fs_protocol(board.fs) == "rsc": # rsc backend uses / for full name dst_sorted = [f"{board.user_name}/{content}" for content in dst_sorted] - if board.fs.protocol == "dbc": + if fs_protocol(board.fs) == "dbc": # TODO: update to match when not read-only dst_sorted = [ "cool_pin", @@ -70,12 +71,12 @@ def test_compat_pin_list(board): def test_compat_pin_versions(board): - if board.fs.protocol == "rsc": + if fs_protocol(board.fs) == "rsc": pytest.skip("RSC uses bundle ids as pin versions") versions = board.pin_versions("df_csv", as_df=False) v_strings = list(v.version for v in versions) # TODO: update when dbc is not read-only - if board.fs.protocol == "dbc": + if fs_protocol(board.fs) == "dbc": v_strings == ["20250410T083026Z-a173c"] else: assert v_strings == ["20220214T163718Z-eceac", "20220214T163720Z-9bfad"] @@ -109,12 +110,12 @@ def test_compat_pin_meta(board): # Note that this fetches the latest of 2 versions meta = board.pin_meta(PIN_CSV) - if board.fs.protocol == "rsc": + if fs_protocol(board.fs) == "rsc": # TODO: afaik the bundle id is largely non-deterministic, so not possible # to test, but should think a bit more about it. assert meta.name == "derek/df_csv" # TODO: update when dbc boards are not read-only - elif board.fs.protocol == "dbc": + elif fs_protocol(board.fs) == "dbc": assert meta.title == "df_csv: a pinned 3 x 2 DataFrame" assert meta.description is None assert meta.created == "20250410T083026Z" @@ -154,7 +155,7 @@ def test_compat_pin_meta_pin_missing(board): def test_compat_pin_meta_version_arg(board): # note that in RSConnect the version is the bundle id # TODO: update when dbc is not read-only - if board.fs.protocol == "dbc": + if fs_protocol(board.fs) == "dbc": meta = board.pin_meta(PIN_CSV, "20250410T083026Z-a173c") assert meta.version.version == "20250410T083026Z-a173c" assert meta.version.hash == "a173c" @@ -185,7 +186,7 @@ def test_compat_pin_read(board): src_df = board.pin_read("df_csv") # TODO: update when dbc boards are not read-only - if board.fs.protocol == "dbc": + if fs_protocol(board.fs) == "dbc": dst_df = pd.DataFrame({"x": [1, 2, 3], "y": [4, 5, 6]}) else: dst_df = pd.read_csv(p_data) @@ -210,7 +211,7 @@ def test_compat_pin_read_supported_rds(board): def test_board_pin_write_manifest_name_error(board_manifest): - if board_manifest.fs.protocol == "rsc": + if fs_protocol(board_manifest.fs) == "rsc": pytest.skip() with pytest.raises(ValueError) as exc_info: diff --git a/pins/tests/test_constructors.py b/pins/tests/test_constructors.py index d0106510..fc63fee2 100644 --- a/pins/tests/test_constructors.py +++ b/pins/tests/test_constructors.py @@ -12,6 +12,7 @@ PATH_TO_EXAMPLE_VERSION, ) from pins.tests.helpers import rm_env, skip_if_dbc +from pins.utils import fs_protocol @pytest.fixture @@ -36,7 +37,7 @@ def check_cache_file_path(p_file, p_cache): def construct_from_board(board): - prot = board.fs.protocol + prot = fs_protocol(board.fs) fs_name = prot if isinstance(prot, str) else prot[0] if fs_name in ["file", ("file", "local")]: @@ -192,7 +193,7 @@ def test_constructor_boards(board, df_csv, tmp_cache): # check data # TODO: update when dbc boards are not read-only - if board.fs.protocol == "dbc": + if fs_protocol(board.fs) == "dbc": pass else: assert_frame_equal(df, df_csv) @@ -200,7 +201,7 @@ def test_constructor_boards(board, df_csv, tmp_cache): # check the cache structure ----------------------------------------------- # check cache - if board.fs.protocol in ["file", ("file", "local")]: + if fs_protocol(board.fs) in ["file", ("file", "local")]: # no caching for local file boards pass else: @@ -239,7 +240,7 @@ def board2(backend): @skip_if_dbc def test_constructor_boards_multi_user(board2, df_csv, tmp_cache): - prot = board2.fs.protocol + prot = fs_protocol(board2.fs) fs_name = prot if isinstance(prot, str) else prot[0] if fs_name == "rsc": @@ -296,7 +297,7 @@ def test_board_constructor_folder(tmp_path: Path, df): def test_board_deparse(board): - prot = board.fs.protocol + prot = fs_protocol(board.fs) with rm_env("CONNECT_API_KEY"): if prot == "rsc": diff --git a/pins/utils.py b/pins/utils.py index 7525c148..4828e54d 100644 --- a/pins/utils.py +++ b/pins/utils.py @@ -20,6 +20,17 @@ def warn_deprecated(msg): warn(msg, DeprecationWarning) +def fs_protocol(fs): + """Return the protocol of a (possibly cache-wrapped) filesystem. + + pins wraps a board's filesystem in a cache filesystem. Since fsspec 2025.9, + a cache filesystem reports its own protocol (e.g. "pinscache") rather than + delegating to the wrapped filesystem, so unwrap it to recover the board's + underlying protocol. + """ + return getattr(fs, "fs", fs).protocol + + def hash_name(path, same_name): if same_name: _hash = os.path.basename(path) diff --git a/pyproject.toml b/pyproject.toml index 14e62a9c..57cd1405 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ requires-python = ">=3.9" dynamic = ["version"] dependencies = [ "appdirs<2", # Using appdirs rather than platformdirs is deliberate, see https://github.com/rstudio/pins-python/pull/239 - "fsspec>=2022.2,<2025.9", + "fsspec>=2022.2", "humanize>=1", "importlib-metadata>=4.4", "importlib-resources>=1.3",