diff --git a/mergin/cli.py b/mergin/cli.py index c20beb9..ed9ad7c 100755 --- a/mergin/cli.py +++ b/mergin/cli.py @@ -248,16 +248,20 @@ def list_projects(ctx, name, namespace, order_params): @click.argument("project") @click.argument("directory", type=click.Path(), required=False) @click.option("--version", default=None, help="Version of project to download") +@click.option("--include", multiple=True, help="Only download files matching this pattern, e.g. '*.gpkg'") +@click.option("--exclude", multiple=True, help="Skip files matching this pattern, e.g. 'media/*'") @click.pass_context -def download(ctx, project, directory, version): +def download(ctx, project, directory, version, include, exclude): """Download last version of mergin project.""" mc = ctx.obj["client"] if mc is None: return + if include and exclude: + raise click.UsageError("--include and --exclude cannot be used together") directory = directory or os.path.basename(project) click.echo("Downloading into {}".format(directory)) try: - job = download_project_async(mc, project, directory, version) + job = download_project_async(mc, project, directory, version, include=include, exclude=exclude) with click.progressbar(length=job.total_size) as bar: last_transferred_size = 0 while download_project_is_running(job): diff --git a/mergin/client.py b/mergin/client.py index 1555051..efe1c8a 100644 --- a/mergin/client.py +++ b/mergin/client.py @@ -63,6 +63,7 @@ from .utils import DateTimeEncoder, get_versions_with_file_changes, int_version, is_version_acceptable from .utils import ( DateTimeEncoder, + filter_files, get_versions_with_file_changes, int_version, is_version_acceptable, @@ -902,7 +903,7 @@ def project_versions(self, project_path, since=1, to=None): filtered_versions = list(filter(lambda v: (num_since <= int_version(v["name"]) <= num_to), versions)) return filtered_versions - def download_project(self, project_path, directory, version=None): + def download_project(self, project_path, directory, version=None, include=None, exclude=None): """ Download project into given directory. If version is not specified, latest version is downloaded @@ -914,8 +915,16 @@ def download_project(self, project_path, directory, version=None): :param version: Project version to download, e.g. v42 :type version: String + + :param include: Optional list of glob patterns (matched against each file's project path, e.g. + "media/*" or "*.gpkg") - only matching files are downloaded. + :type include: List[String] + + :param exclude: Optional list of glob patterns - matching files are skipped. Mutually exclusive + with include. + :type exclude: List[String] """ - job = download_project_async(self, project_path, directory, version) + job = download_project_async(self, project_path, directory, version, include=include, exclude=exclude) download_project_wait(job) download_project_finalize(job) @@ -1158,6 +1167,10 @@ def project_status(self, directory): server_info = self.project_info(mp.project_full_name(), since=mp.version()) pull_changes = mp.get_pull_changes(server_info.get("files", []), server_info.get("version")) + # on a sparse checkout, don't report excluded files as pending server changes - + # they were never meant to be pulled in the first place + file_filter = mp.file_filter() + pull_changes = {change_type: filter_files(files, **file_filter) for change_type, files in pull_changes.items()} push_changes = mp.get_push_changes() push_changes_summary = mp.get_list_of_push_changes(push_changes) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 5210089..6805dd2 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -25,7 +25,7 @@ from .common import CHUNK_SIZE, ClientError, DeltaChangeType, PullActionType from .models import ProjectDelta, ProjectDeltaChange, PullAction from .merginproject import MerginProject -from .utils import cleanup_tmp_dir, save_to_file +from .utils import cleanup_tmp_dir, filter_files, path_matches_filter, save_to_file, validates_file_filter from typing import List, Optional # status = download_project_async(...) @@ -242,10 +242,15 @@ def _cleanup_failed_download(mergin_project: MerginProject = None): return dest_path -def download_project_async(mc, project_path, directory, project_version=None): +@validates_file_filter +def download_project_async(mc, project_path, directory, project_version=None, include=None, exclude=None): """ Starts project download in background and returns handle to the pending project download. Using that object it is possible to watch progress or cancel the ongoing work. + + `include`/`exclude` are optional lists of glob patterns (matched against each file's project + path, e.g. "media/*" or "*.gpkg") to only download a subset of the project's files. They are + mutually exclusive. """ if "/" not in project_path: @@ -276,6 +281,11 @@ def download_project_async(mc, project_path, directory, project_version=None): mp.log.info(f"got project info. version {version}") + # keep only the files matching the filter (if any) + project_info["files"] = filter_files(project_info["files"], include=include, exclude=exclude) + if include or exclude: + project_info["file_filter"] = {"include": include, "exclude": exclude} + # prepare download update_tasks = [] # stuff to do at the end of download for file in project_info["files"]: @@ -525,6 +535,9 @@ def pull_project_async(mc, directory) -> Optional[PullJob]: mp.log.info("--- pull aborted") raise + file_filter = mp.file_filter() + delta.changes = [c for c in delta.changes if path_matches_filter(c.path, **file_filter)] + mp.log.info(f"got project versions: local version {local_version} / server version {server_version}") if local_version == server_version: @@ -748,6 +761,14 @@ def pull_project_finalize(job: PullJob): cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content raise ClientError("Failed to apply pull actions: " + str(e)) + file_filter = job.mp.file_filter() + job.project_info["files"] = filter_files(job.project_info["files"], **file_filter) + # keep the sparse checkout: re-apply the filter this project was downloaded with, + # since job.project_info is a fresh, unfiltered response from the server + # and update_metadata() replaces the whole metadata dict rather than merging into it + if file_filter["include"] or file_filter["exclude"]: + job.project_info["file_filter"] = file_filter + job.mp.update_metadata(job.project_info) if job.mp.has_unfinished_pull(): diff --git a/mergin/client_push.py b/mergin/client_push.py index 831b59b..1c9d3d7 100644 --- a/mergin/client_push.py +++ b/mergin/client_push.py @@ -34,7 +34,7 @@ ) from .merginproject import MerginProject, pygeodiff from .editor import filter_changes -from .utils import get_data_checksum, cleanup_tmp_dir +from .utils import get_data_checksum, cleanup_tmp_dir, filter_files POST_JSON_HEADERS = {"Content-Type": "application/json"} @@ -458,6 +458,14 @@ def push_project_finalize(job: UploadJob): cleanup_tmp_dir(job.mp, job.tmp_dir) # delete our temporary dir and all its content raise err + # keep the sparse checkout: re-apply the filter this project was + # downloaded with, since job.server_resp is a fresh, unfiltered response from the server + # and update_metadata() replaces the whole metadata dict rather than merging into it + file_filter = job.mp.file_filter() + job.server_resp["files"] = filter_files(job.server_resp["files"], **file_filter) + if file_filter["include"] or file_filter["exclude"]: + job.server_resp["file_filter"] = file_filter + job.mp.update_metadata(job.server_resp) try: job.mp.apply_push_changes(asdict(job.changes)) diff --git a/mergin/merginproject.py b/mergin/merginproject.py index 12d798f..b4544e7 100644 --- a/mergin/merginproject.py +++ b/mergin/merginproject.py @@ -24,6 +24,7 @@ unique_path_name, conflicted_copy_file_name, edit_conflict_file_name, + filter_files, ) from .local_changes import FileChange @@ -215,6 +216,13 @@ def files(self) -> list: self._read_metadata() return self._metadata["files"] + def file_filter(self) -> dict: + """ + Returns the include/exclude file filter this project was downloaded with, as a dict with "include" and "exclude" keys. + """ + self._read_metadata() + return self._metadata.get("file_filter", {"include": None, "exclude": None}) + @property def metadata(self) -> dict: """Gets raw access to metadata. Kept only for backwards compatibility and will be removed.""" @@ -566,7 +574,8 @@ def get_local_delta(self, diff_directory: str) -> List[ProjectDeltaChange]: :rtype: List[ProjectDeltaItem] """ result = [] - changes = self.compare_file_sets(self.files(), self.inspect_files()) + current_files = filter_files(self.inspect_files(), **self.file_filter()) + changes = self.compare_file_sets(self.files(), current_files) added = changes.get("added", []) removed = changes.get("removed", []) updated = changes.get("updated", []) @@ -655,7 +664,8 @@ def get_push_changes(self): :returns: changes metadata for files to be pushed to server :rtype: dict """ - changes = self.compare_file_sets(self.files(), self.inspect_files()) + current_files = filter_files(self.inspect_files(), **self.file_filter()) + changes = self.compare_file_sets(self.files(), current_files) # do checkpoint to push changes from wal file to gpkg for file in changes["added"] + changes["updated"]: size, checksum = do_sqlite_checkpoint(self.fpath(file["path"]), self.log) diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index bd987d0..93e9153 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -1,4 +1,5 @@ import hashlib +import inspect import json import logging import os @@ -1153,6 +1154,270 @@ def test_download_versions(mc): mc.download_project(project, project_dir_v3, "v3") +def test_download_project_with_filter(mc): + """Test downloading a project with include/exclude filters, and that they're mutually exclusive.""" + test_project = "test_download_project_filter" + project = create_project_path(test_project, mc) + project_dir = os.path.join(TMP_DIR, test_project) + include_dir = os.path.join(TMP_DIR, test_project + "_include") + exclude_dir = os.path.join(TMP_DIR, test_project + "_exclude") + conflict_dir = os.path.join(TMP_DIR, test_project + "_conflict") + + cleanup(mc, project, [project_dir, include_dir, exclude_dir, conflict_dir]) + shutil.copytree(TEST_DATA_DIR, project_dir) + mc.create_project_and_push(project, project_dir) + + # include filter: only matching files are fetched + mc.download_project(project, include_dir, include=["*.gpkg"]) + + downloaded_files = set() + for root, _, files in os.walk(include_dir): + if ".mergin" in root.split(os.sep): + continue + for f in files: + rel = os.path.relpath(os.path.join(root, f), include_dir) + downloaded_files.add(rel.replace(os.sep, "/")) + + assert downloaded_files + assert all(f.endswith(".gpkg") for f in downloaded_files) + assert "test.qgs" not in downloaded_files + assert "test.txt" not in downloaded_files + + mp = MerginProject(include_dir) + assert all(f["path"].endswith(".gpkg") for f in mp.files()) + assert mp.file_filter() == {"include": ["*.gpkg"], "exclude": None} + + # exclude filter: matching files are skipped + mc.download_project(project, exclude_dir, exclude=["test_dir/*"]) + + assert os.path.exists(os.path.join(exclude_dir, "base.gpkg")) + assert not os.path.exists(os.path.join(exclude_dir, "test_dir")) + + mp = MerginProject(exclude_dir) + assert not any(f["path"].startswith("test_dir/") for f in mp.files()) + assert mp.file_filter() == {"include": None, "exclude": ["test_dir/*"]} + + # include and exclude cannot be combined + with pytest.raises(ClientError, match="Cannot use both include and exclude"): + mc.download_project(project, conflict_dir, include=["*.gpkg"], exclude=["*.txt"]) + assert not os.path.exists(conflict_dir) + + +def test_sparse_checkout_filter_is_immutable(mc): + """Once a directory has been downloaded with a filter, there is no way to change that + filter in place - the only way to get a different filter is to check out into a fresh + directory. + """ + test_project = "test_sparse_checkout_immutable" + project = create_project_path(test_project, mc) + project_dir = os.path.join(TMP_DIR, test_project) + sparse_dir = os.path.join(TMP_DIR, test_project + "_sparse") + + cleanup(mc, project, [project_dir, sparse_dir]) + shutil.copytree(TEST_DATA_DIR, project_dir) + mc.create_project_and_push(project, project_dir) + + mc.download_project(project, sparse_dir, exclude=["test_dir/*"]) + original_filter = MerginProject(sparse_dir).file_filter() + + # trying to download again into the same directory - with the same or a different filter - + # must fail without touching anything, regardless of what filter (if any) is requested + for kwargs in ({"exclude": ["test_dir/*"]}, {"include": ["*.gpkg"]}, {}): + with pytest.raises(ClientError, match="Project directory already exists"): + mc.download_project(project, sparse_dir, **kwargs) + + assert MerginProject(sparse_dir).file_filter() == original_filter + + # pull_project()/push_project() take no filter arguments at all - there is no API through + # which a different filter could be supplied for an existing checkout + assert "include" not in inspect.signature(mc.pull_project).parameters + assert "include" not in inspect.signature(mc.push_project).parameters + + +def test_pull_on_sparse_checkout(mc): + """A sparse checkout keeps respecting its filter across subsequent pulls: excluded files + stay ignored even as the server moves ahead, and a genuine local-vs-server conflict on an + included file is still detected and resolved correctly. + """ + test_project = "test_pull_sparse_checkout" + project = create_project_path(test_project, mc) + project_dir = os.path.join(TMP_DIR, test_project) + sparse_dir = os.path.join(TMP_DIR, test_project + "_sparse") + + cleanup(mc, project, [project_dir, sparse_dir]) + create_versioned_project(mc, test_project, project_dir, "base.gpkg", remove=False) + + mc.download_project(project, sparse_dir, exclude=["test_dir/*"]) + assert not os.path.exists(os.path.join(sparse_dir, "test_dir")) + sparse_version_before = MerginProject(sparse_dir).version() + + # change an excluded file on the server, by pushing from the reference (full) checkout + mp_ref = MerginProject(project_dir) + with open(mp_ref.fpath("test_dir/test2.txt"), "a") as f: + f.write("change that only affects an excluded file") + mc.push_project(project_dir) + server_version = mc.project_info(project)["version"] + assert server_version != sparse_version_before + + mc.pull_project(sparse_dir) + + mp_sparse = MerginProject(sparse_dir) + assert mp_sparse.version() == server_version + assert not os.path.exists(os.path.join(sparse_dir, "test_dir")) + assert not any(f["path"].startswith("test_dir/") for f in mp_sparse.files()) + + # now a genuine conflict: edit an included file locally, but don't push it yet + shutil.copy(os.path.join(TEST_DATA_DIR, "two_tables.gpkg"), os.path.join(sparse_dir, "base.gpkg")) + + # meanwhile a conflicting edit to the same file gets pushed from the reference (full) checkout + shutil.copy(os.path.join(TEST_DATA_DIR, "two_tables_drop.gpkg"), os.path.join(project_dir, "base.gpkg")) + mc.push_project(project_dir) + server_version = mc.project_info(project)["version"] + assert server_version != mp_sparse.version() + + # pulling the sparse checkout now must detect the conflict (not silently drop the local + # edit, not crash, and not leave the sparse checkout in a broken state) + mc.pull_project(sparse_dir) + + mp_sparse = MerginProject(sparse_dir) + assert mp_sparse.version() == server_version + assert not os.path.exists(os.path.join(sparse_dir, "test_dir")) # filter still respected + conflict_files = [f for f in os.listdir(sparse_dir) if "conflicted copy" in f] + assert conflict_files, "expected a conflicted copy of base.gpkg to be created" + + +def test_push_on_sparse_checkout(mc): + """Pushing from a sparse checkout must never touch the files it never downloaded: not a + no-op push (must not delete excluded files it's not tracking), not a real push of an + included file's edit (must not wipe the persisted filter from local metadata. + """ + test_project = "test_push_sparse_checkout" + project = create_project_path(test_project, mc) + project_dir = os.path.join(TMP_DIR, test_project) + sparse_dir = os.path.join(TMP_DIR, test_project + "_sparse") + + cleanup(mc, project, [project_dir, sparse_dir]) + create_versioned_project(mc, test_project, project_dir, "base.gpkg", remove=False) + + mc.download_project(project, sparse_dir, exclude=["test_dir/*"]) + + # no local edits - must be a no-op, not a deletion of excluded files + mc.push_project(sparse_dir) + + server_files = {f["path"] for f in mc.project_info(project)["files"]} + assert "test_dir/test2.txt" in server_files + assert "test_dir/modified_1_geom.gpkg" in server_files + + # a real push of an included file's edit must not lose the persisted filter afterwards + shutil.copy(os.path.join(TEST_DATA_DIR, "two_tables_drop.gpkg"), os.path.join(sparse_dir, "base.gpkg")) + mc.push_project(sparse_dir) + + mp_sparse = MerginProject(sparse_dir) + assert mp_sparse.file_filter() == {"include": None, "exclude": ["test_dir/*"]} + assert not any(f["path"].startswith("test_dir/") for f in mp_sparse.files()) + server_files = {f["path"] for f in mc.project_info(project)["files"]} + assert "test_dir/test2.txt" in server_files + assert "test_dir/modified_1_geom.gpkg" in server_files + + # manually add a file outside the filter's scope - it must not get pushed either + os.makedirs(os.path.join(sparse_dir, "test_dir"), exist_ok=True) + with open(os.path.join(sparse_dir, "test_dir", "new_stray.txt"), "w") as f: + f.write("should not be pushed") + + mc.push_project(sparse_dir) + + server_files = {f["path"] for f in mc.project_info(project)["files"]} + assert "test_dir/new_stray.txt" not in server_files + + # a delete-only push goes through a different shortcut code path server-side + # it must also keep the filter intact and leave excluded files alone + # old_metadata.json already present on server and not affected by filter + os.remove(os.path.join(sparse_dir, "old_metadata.json")) + mc.push_project(sparse_dir) + + mp_sparse = MerginProject(sparse_dir) + assert mp_sparse.file_filter() == {"include": None, "exclude": ["test_dir/*"]} + assert not any(f["path"].startswith("test_dir/") for f in mp_sparse.files()) + server_files = {f["path"] for f in mc.project_info(project)["files"]} + assert "old_metadata.json" not in server_files + assert "test_dir/test2.txt" in server_files + assert "test_dir/modified_1_geom.gpkg" in server_files + + +def test_sparse_checkout_pull_push_v1_api(mc): + """Same filter-persistence guarantees as test_pull_on_sparse_checkout/test_push_on_sparse_checkout, + but forcing the legacy v1 pull/push code paths. + """ + server_features = mc.server_features() + mc._server_features = {"v2_pull_enabled": False, "v2_push_enabled": False} + + test_project = "test_sparse_checkout_v1_api" + project = create_project_path(test_project, mc) + project_dir = os.path.join(TMP_DIR, test_project) + sparse_dir = os.path.join(TMP_DIR, test_project + "_sparse") + + cleanup(mc, project, [project_dir, sparse_dir]) + create_versioned_project(mc, test_project, project_dir, "base.gpkg", remove=False) + + mc.download_project(project, sparse_dir, exclude=["test_dir/*"]) + + # change an excluded file on the server - a v1 pull must still skip it + mp_ref = MerginProject(project_dir) + with open(mp_ref.fpath("test_dir/test2.txt"), "a") as f: + f.write("change to an excluded file, v1 api") + mc.push_project(project_dir) + + mc.pull_project(sparse_dir) + + mp_sparse = MerginProject(sparse_dir) + assert mp_sparse.version() == mc.project_info(project)["version"] + assert not os.path.exists(os.path.join(sparse_dir, "test_dir")) + assert not any(f["path"].startswith("test_dir/") for f in mp_sparse.files()) + + # a real edit to an included file, pushed via v1 - filter must survive it + shutil.copy(os.path.join(TEST_DATA_DIR, "two_tables_drop.gpkg"), os.path.join(sparse_dir, "base.gpkg")) + mc.push_project(sparse_dir) + + mp_sparse = MerginProject(sparse_dir) + assert mp_sparse.file_filter() == {"include": None, "exclude": ["test_dir/*"]} + assert not any(f["path"].startswith("test_dir/") for f in mp_sparse.files()) + server_files = {f["path"] for f in mc.project_info(project)["files"]} + assert "test_dir/test2.txt" in server_files + assert "test_dir/modified_1_geom.gpkg" in server_files + + mc._server_features = server_features + + +def test_project_status_on_sparse_checkout(mc): + """`status` must not report excluded files as pending server changes - + they were deliberately never meant to be pulled, so they shouldn't show up as if a pull + were needed to fetch them. + """ + test_project = "test_status_sparse_checkout" + project = create_project_path(test_project, mc) + project_dir = os.path.join(TMP_DIR, test_project) + sparse_dir = os.path.join(TMP_DIR, test_project + "_sparse") + + cleanup(mc, project, [project_dir, sparse_dir]) + create_versioned_project(mc, test_project, project_dir, "base.gpkg", remove=False) + + mc.download_project(project, sparse_dir, exclude=["test_dir/*"]) + + # change both an excluded and an included file on the server + mp_ref = MerginProject(project_dir) + with open(mp_ref.fpath("test_dir/test2.txt"), "a") as f: + f.write("change to an excluded file") + shutil.copy(os.path.join(TEST_DATA_DIR, "two_tables_drop.gpkg"), os.path.join(project_dir, "base.gpkg")) + mc.push_project(project_dir) + + pull_changes, _, _ = mc.project_status(sparse_dir) + + changed_paths = {f["path"] for files in pull_changes.values() for f in files} + assert "test_dir/test2.txt" not in changed_paths + # a real, included change must still be reported + assert "base.gpkg" in changed_paths + + def test_paginated_project_list(mc): """Test the new endpoint for projects list with pagination, ordering etc.""" test_projects = dict() diff --git a/mergin/test/test_mergin_project.py b/mergin/test/test_mergin_project.py index 97fe554..b6879ea 100644 --- a/mergin/test/test_mergin_project.py +++ b/mergin/test/test_mergin_project.py @@ -186,6 +186,7 @@ def test_get_local_delta(): # Mock files() to return origin info for version lookup mp.files = lambda: [] mp.inspect_files = lambda: [] # Dummy return + mp.file_filter = lambda: {"include": None, "exclude": None} # no sparse checkout filter in this test # check if geopackage is updated (is_open) but missing - geodiff lib error, than updated file is reported mock_changes = { diff --git a/mergin/test/test_utils.py b/mergin/test/test_utils.py new file mode 100644 index 0000000..973db25 --- /dev/null +++ b/mergin/test/test_utils.py @@ -0,0 +1,80 @@ +import pytest + +from ..utils import path_matches_filter, filter_files +from ..common import ClientError + + +@pytest.mark.parametrize( + "path, include, exclude, expected", + [ + # no filter at all -> everything kept + pytest.param("anything/at/all.gpkg", None, None, True, id="no-filter"), + pytest.param("data.gpkg", [], None, True, id="empty-include-list"), + pytest.param("data.gpkg", None, [], True, id="empty-exclude-list"), + # basic include/exclude + pytest.param("data.gpkg", ["*.gpkg"], None, True, id="include-match"), + pytest.param("data.txt", ["*.gpkg"], None, False, id="include-no-match"), + pytest.param("media/photo.jpg", None, ["media/*"], False, id="exclude-match"), + pytest.param("data.gpkg", None, ["media/*"], True, id="exclude-no-match"), + # subfolders: fnmatch's '*' crosses '/', so it reaches arbitrarily deep + pytest.param("media/photo.jpg", None, ["media/*"], False, id="subfolder-direct-child"), + pytest.param("media/sub/deep/photo.jpg", None, ["media/*"], False, id="subfolder-deeply-nested"), + pytest.param("layer.gpkg", ["*.gpkg"], None, True, id="extension-pattern-at-root"), + pytest.param("data/nested/deep/layer.gpkg", ["*.gpkg"], None, True, id="extension-pattern-at-any-depth"), + # patterns still anchor to the *full* path, not just the basename + pytest.param("nested/media/photo.jpg", None, ["media/*"], True, id="not-anchored-to-basename-kept"), + pytest.param("nested/media/photo.jpg", None, ["*/media/*"], False, id="leading-star-catches-nested-media"), + pytest.param("media/photo.jpg", None, ["*/media/*"], True, id="leading-star-misses-root-level-media"), + # case sensitivity: fnmatchcase, not fnmatch - always case-sensitive, any OS + pytest.param("data.GPKG", ["*.gpkg"], None, False, id="case-mismatch-in-path"), + pytest.param("data.gpkg", ["*.GPKG"], None, False, id="case-mismatch-in-pattern"), + pytest.param("Media/photo.jpg", None, ["media/*"], True, id="case-mismatch-in-directory-kept"), + # lists of patterns: a path matches if it matches ANY pattern in the list (OR) + pytest.param("data.gpkg", ["*.gpkg", "*.qgz", "project.qgs"], None, True, id="include-list-1st-matches"), + pytest.param("map.qgz", ["*.gpkg", "*.qgz", "project.qgs"], None, True, id="include-list-2nd-matches"), + pytest.param("project.qgs", ["*.gpkg", "*.qgz", "project.qgs"], None, True, id="include-list-3rd-matches"), + pytest.param("readme.txt", ["*.gpkg", "*.qgz", "project.qgs"], None, False, id="include-list-none-match"), + pytest.param("media/photo.jpg", None, ["media/*", "*.tmp", "*-wal"], False, id="exclude-list-1st-matches"), + pytest.param("scratch.tmp", None, ["media/*", "*.tmp", "*-wal"], False, id="exclude-list-2nd-matches"), + pytest.param("data.gpkg-wal", None, ["media/*", "*.tmp", "*-wal"], False, id="exclude-list-3rd-matches"), + pytest.param("data.gpkg", None, ["media/*", "*.tmp", "*-wal"], True, id="exclude-list-none-match"), + ], +) +def test_path_matches_filter(path, include, exclude, expected): + assert path_matches_filter(path, include=include, exclude=exclude) is expected + + +@pytest.mark.parametrize( + "include, exclude, expected_paths", + [ + pytest.param(None, None, {"a.gpkg", "b.qgz", "c.txt", "media/d.gpkg"}, id="no-filter"), + pytest.param(["*.gpkg", "*.qgz"], None, {"a.gpkg", "b.qgz", "media/d.gpkg"}, id="include-list"), + pytest.param(None, ["media/*"], {"a.gpkg", "b.qgz", "c.txt"}, id="exclude-subfolder"), + ], +) +def test_filter_files(include, exclude, expected_paths): + files = [{"path": p} for p in ["a.gpkg", "b.qgz", "c.txt", "media/d.gpkg"]] + result = filter_files(files, include=include, exclude=exclude) + assert {f["path"] for f in result} == expected_paths + + +def test_filter_files_keeps_matching_dicts_as_is(): + """filter_files() passes matching dicts through unchanged""" + files = [ + {"path": "project.gpkg", "size": 100}, + {"path": "media/photo.jpg", "size": 999}, + ] + + result = filter_files(files, exclude=["media/*"]) + + assert result == [{"path": "project.gpkg", "size": 100}] + assert result[0] is files[0] + + +def test_filter_files_raises_on_mutually_exclusive_args(): + """Unlike path_matches_filter, filter_files() is a public entry point (decorated with + @validates_file_filter) and does enforce that include/exclude are mutually exclusive. + """ + files = [{"path": "a.gpkg"}] + with pytest.raises(ClientError, match="Cannot use both include and exclude"): + filter_files(files, include=["*.gpkg"], exclude=["*.txt"]) diff --git a/mergin/utils.py b/mergin/utils.py index 91796f3..1b7a234 100644 --- a/mergin/utils.py +++ b/mergin/utils.py @@ -2,13 +2,16 @@ import io import json import hashlib +import fnmatch +import functools +import inspect import re import sqlite3 from datetime import datetime from pathlib import Path import tempfile from enum import Enum -from typing import Optional, Type, Union, ByteString +from typing import List, Optional, Type, Union, ByteString from .common import ClientError @@ -280,6 +283,53 @@ def is_mergin_config(path: str) -> bool: return filename == "mergin-config.json" +def validates_file_filter(func): + """ + Marks a function as accepting an `include`/`exclude` glob-filter signature, and validates + those arguments (mutually exclusive) before every call - regardless of whether the caller + passed them positionally or by keyword. + """ + signature = inspect.signature(func) + + @functools.wraps(func) + def wrapper(*args, **kwargs): + bound_args = signature.bind_partial(*args, **kwargs) + if bound_args.arguments.get("include") and bound_args.arguments.get("exclude"): + raise ClientError("Cannot use both include and exclude filters at the same time") + return func(*args, **kwargs) + + return wrapper + + +def path_matches_filter(path: str, include: List[str] = None, exclude: List[str] = None) -> bool: + """ + Returns whether `path` should be kept under a sparse-checkout style include/exclude filter. + + With `include`, only paths matching at least one glob pattern are kept. With `exclude`, + paths matching at least one pattern are dropped. With neither given, every path is kept. + """ + if include: + return any(fnmatch.fnmatchcase(path, pattern) for pattern in include) + if exclude: + return not any(fnmatch.fnmatchcase(path, pattern) for pattern in exclude) + return True + + +@validates_file_filter +def filter_files(files: List[dict], include: List[str] = None, exclude: List[str] = None) -> List[dict]: + """ + Keep only files matching a sparse-checkout style filter. + + :param files: list of file metadata dicts, each with a 'path' key + :param include: glob patterns - only matching files are kept + :param exclude: glob patterns - matching files are dropped + :returns: filtered list of file metadata dicts + + .. seealso:: path_matches_filter + """ + return [f for f in files if path_matches_filter(f["path"], include=include, exclude=exclude)] + + def bytes_to_human_size(bytes: int): """ Convert bytes to human readable size