fixing decimation

Author: admclachlan

requirements.txt

@@ -11,7 +11,13 @@ certifi
 flybrains>=0.6.3      # BANC support arrived in 0.6.x; 0.3.0 has none
 navis                 # transform registry + xform_brain
 trimesh               # mesh IO and plane slicing
-fast-simplification   # quadric decimation of the served OBJs (ISSUES.md IMG-1)
+# Quadric decimation of the served OBJs (docs/ISSUES.md IMG-1). MUST be version-marked:
+# 0.2.0's metadata says requires-python >=3.9 but its code uses PEP 604 unions
+# (`float | None` in simplify.py), which is a SyntaxError-at-import on 3.9. Every 0.1.x is
+# clean. pyproject.toml reads this file as its base dependencies, so an unpinned entry here
+# overrides the marker in the [images] extra — pin it in both places or neither works.
+fast-simplification<0.2 ; python_version < "3.10"
+fast-simplification     ; python_version >= "3.10"
 shapely               # required by trimesh.slice_plane, even with cap=False
 scipy                 # map_coordinates for the baked-field lookup
 cloud-volume          # BANC precomputed mesh fetch

src/.DS_Store

@@ -26,11 +26,22 @@
 #: product key -> served filename. The keys are what `--products` accepts.
 PRODUCTS = {'swc': 'volume.swc', 'obj': 'volume_man.obj', 'nrrd': 'volume.nrrd'}
 
-#: Globs for everything this loader considers "the served image". Thumbnails are included
-#: because a thumbnail depicts the OLD alignment: leaving one beside a replaced volume is a
-#: silently stale image.
+#: Globs for everything VFB serves per neuron. Used to decide whether an image exists at
+#: all, and to clear one that turns out to be spurious — NOT to decide what a successful
+#: replacement sweeps away (see SWEEP_AFTER_SWAP).
+#: Confirmed served 2026-08-26: volume.swc, volume.nrrd, volume_man.obj, volume.obj,
+#: volume.wlz, thumbnail.png, thumbnailT.png.
 SERVED_GLOBS = ('volume*', 'thumbnail*')
 
+#: Extra files removed after a successful swap, beyond the products this run wrote.
+#: Decided 2026-08-26: replace the three we generate, additionally delete `volume.obj`,
+#: and **leave everything else** — `volume.wlz`, `thumbnail*` and anything new. Those will
+#: be briefly out of sync with the new alignment, which is accepted: other jobs refresh
+#: them in time, and the alternative (deleting products nothing here regenerates) is worse.
+#: An earlier version swept `volume*` + `thumbnail*` wholesale, which removed 5 files per
+#: neuron including `volume.wlz`.
+SWEEP_AFTER_SWAP = ('volume.obj',)
+
 #: Statuses meaning "this neuron has been dealt with; do not redo it on resume".
 #: 'error' is deliberately absent — errors must be retried.
 TERMINAL = frozenset({'replaced', 'created', 'deleted_spurious', 'skipped',
@@ -94,12 +105,23 @@ def swap(self, built):
 
         Returns (wrote, removed) as basename lists. `os.replace` overwrites, so there is
         no delete-then-write step and no window in which a served file is missing.
+
+        The sweep is **narrow and explicit**: `SWEEP_AFTER_SWAP`, plus any product this
+        loader manages that this run did not write (so `--products swc,nrrd` does not
+        leave last alignment's `volume_man.obj` behind). Everything else VFB serves is
+        left untouched — see the SWEEP_AFTER_SWAP note.
         """
         wrote = []
         for tmp, final in built.items():
             os.replace(tmp, final)
             wrote.append(os.path.basename(final))
-        removed = self._sweep(keep=set(built.values()))
+
+        just_wrote = {os.path.basename(p) for p in built.values()}
+        targets = set(SWEEP_AFTER_SWAP) | set(PRODUCTS.values())
+        removed = []
+        for name in sorted(targets - just_wrote):
+            if _unlink(os.path.join(self.folder, name)):
+                removed.append(name)
         return sorted(wrote), removed
 
     def archive_to(self, dest):

src/vfb_connectomics_import/images/io.py

@@ -133,7 +133,7 @@ def __init__(self, npy_path, mmap=True):
 
     def __repr__(self):
         return (f'<BakedField {os.path.basename(self.path)} -> {self.target}, '
-                f'grid {tuple(self.shape)} @ {self.step[0]:.0f} nm>')
+                f'grid {tuple(int(n) for n in self.shape)} @ {self.step[0]:.0f} nm>')   # int(): numpy 2 reprs bare scalars as np.int64(445)
 
     def __call__(self, pts):
         pts = np.asarray(pts, float)

src/vfb_connectomics_import/images/transforms.py

@@ -19,21 +19,32 @@
     os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'src'))
 
 from vfb_connectomics_import.images.io import (
-    PRODUCTS, TERMINAL, Ledger, OutputSet, partial_path)
+    PRODUCTS, SWEEP_AFTER_SWAP, TERMINAL, Ledger, OutputSet, partial_path)
 
 ALL = ('swc', 'obj', 'nrrd')
 
 
 # --------------------------------------------------------------------------- helpers
-def plant(folder, thumbnail=True, volumes=ALL):
+#: What VFB actually serves per neuron, confirmed live 2026-08-26. Only the first three
+#: are written by this loader; the rest belong to other jobs.
+SERVED = ('volume.swc', 'volume.nrrd', 'volume_man.obj',
+          'volume.obj', 'volume.wlz', 'thumbnail.png', 'thumbnailT.png')
+
+
+def plant(folder, thumbnail=True, volumes=ALL, extras=True):
     """An existing v626-era image, as the loader will find on almost every neuron."""
     os.makedirs(folder, exist_ok=True)
     for k in volumes:
         with open(os.path.join(folder, PRODUCTS[k]), 'w') as fh:
             fh.write(f'OLD v626 {k}')
+    if extras:
+        for n in ('volume.obj', 'volume.wlz'):
+            with open(os.path.join(folder, n), 'w') as fh:
+                fh.write('OLD ' + n)
     if thumbnail:
-        with open(os.path.join(folder, 'thumbnail.png'), 'w') as fh:
-            fh.write('OLD THUMB')
+        for n in ('thumbnail.png', 'thumbnailT.png'):
+            with open(os.path.join(folder, n), 'w') as fh:
+                fh.write('OLD THUMB')
 
 
 def names(folder):
@@ -66,16 +77,27 @@ def test_partial_suffix_precedes_the_extension():
 
 
 # ------------------------------------------------------------------------ the swap is safe
-def test_swap_replaces_content_and_clears_thumbnail():
+def test_swap_replaces_three_products_and_deletes_only_volume_obj():
+    """The agreed contract (2026-08-26): replace swc / nrrd / volume_man.obj, additionally
+    delete volume.obj, and LEAVE volume.wlz and the thumbnails. Those go briefly out of
+    sync with the new alignment, which is accepted -- other jobs refresh them, and
+    deleting products nothing here regenerates would be worse."""
     with tempfile.TemporaryDirectory() as d:
         plant(d)
         out = OutputSet(d, ALL)
         assert read(d, 'swc') == 'OLD v626 swc'
         wrote, removed = out.swap(build(out, ALL))
         assert wrote == sorted(PRODUCTS[k] for k in ALL)
-        assert removed == ['thumbnail.png'], 'a stale thumbnail depicts the old alignment'
+        assert removed == ['volume.obj'], removed
         assert read(d, 'swc') == 'NEW swc'
-        assert 'thumbnail.png' not in names(d)
+        for n in ('volume.wlz', 'thumbnail.png', 'thumbnailT.png'):
+            assert n in names(d), n + ' must NOT be swept'
+
+
+def test_sweep_list_is_exactly_volume_obj():
+    """Pinned because an earlier version globbed volume*/thumbnail* and removed five files
+    per neuron, including volume.wlz, which nothing in this repo regenerates."""
+    assert SWEEP_AFTER_SWAP == ('volume.obj',)
 
 
 def test_swap_never_leaves_a_served_file_missing():
@@ -99,8 +121,8 @@ def test_swap_sweeps_a_product_dropped_from_products():
         plant(d, thumbnail=False)
         out = OutputSet(d, ('swc', 'nrrd'))
         _, removed = out.swap(build(out, ('swc', 'nrrd')))
-        assert removed == ['volume_man.obj']
-        assert names(d) == ['volume.nrrd', 'volume.swc']
+        assert removed == ['volume.obj', 'volume_man.obj'], removed
+        assert 'volume.wlz' in names(d), 'still not ours to delete'
 
 
 # ------------------------------------------------------------- existing-image bookkeeping
@@ -138,8 +160,8 @@ def test_remove_all_clears_volumes_and_thumbnails():
     with tempfile.TemporaryDirectory() as d:
         plant(d)
         removed = OutputSet(d, ALL).remove_all()
-        assert removed == ['thumbnail.png', 'volume.nrrd', 'volume.swc', 'volume_man.obj']
-        assert names(d) == []
+        assert sorted(removed) == sorted(SERVED), removed
+        assert names(d) == [], 'a spurious image must not be left half-served'
 
 
 # --------------------------------------------------------------------- the deletion policy