diff --git a/src/datajoint/jobs.py b/src/datajoint/jobs.py index 97cfafb15..13832c9f2 100644 --- a/src/datajoint/jobs.py +++ b/src/datajoint/jobs.py @@ -13,7 +13,7 @@ import platform import subprocess -from .condition import AndList +from .condition import AndList, Not from .errors import DataJointError, DuplicateError from .heading import Heading from .table import Table @@ -370,8 +370,6 @@ def refresh( # Keys that need jobs: in key_source, not in target, not in jobs # Disable semantic_check for Job table (self) because its attributes may not have matching lineage - from .condition import Not - new_keys = (key_source - self._target).restrict(Not(self), semantic_check=False).proj() new_key_list = new_keys.keys() @@ -395,8 +393,10 @@ def refresh( # 2. Re-pend success jobs if keep_completed=True if config.jobs.keep_completed: # Success jobs whose keys are in key_source but not in target - # Disable semantic_check for Job table operations - success_to_repend = self.completed.restrict(key_source, semantic_check=False) - self._target + # Disable semantic_check for Job table operations (job table PK has different lineage than target) + success_to_repend = self.completed.restrict(key_source, semantic_check=False).restrict( + Not(self._target), semantic_check=False + ) repend_keys = success_to_repend.keys() for key in repend_keys: (self & key).delete_quick() diff --git a/tests/integration/test_jobs.py b/tests/integration/test_jobs.py index bc00cf0f8..20fa3233d 100644 --- a/tests/integration/test_jobs.py +++ b/tests/integration/test_jobs.py @@ -158,3 +158,52 @@ def test_long_error_stack(clean_jobs, subject, experiment): experiment.jobs.error(key, "error message", long_error_stack) error_stack = experiment.jobs.errors.fetch1("error_stack") assert error_stack == long_error_stack, "error stacks do not agree" + + +def test_populate_reserve_jobs_with_keep_completed(clean_jobs, subject, experiment): + """Test populate(reserve_jobs=True) with keep_completed=True. + + Regression test for https://github.com/datajoint/datajoint-python/issues/1379 + The bug was that the `-` operator in jobs.refresh() didn't pass semantic_check=False, + causing a DataJointError about different lineages when keep_completed=True. + """ + # Clear experiment data to ensure there's work to do + experiment.delete() + + with dj.config.override(jobs={"keep_completed": True, "add_job_metadata": True}): + # Should not raise DataJointError about semantic matching + experiment.populate(reserve_jobs=True) + + # Verify jobs completed successfully + assert len(experiment) > 0, "No data was populated" + assert len(experiment.jobs.errors) == 0, "Unexpected errors during populate" + + # With keep_completed=True, completed jobs should be retained + assert len(experiment.jobs.completed) > 0, "Completed jobs not retained" + + +def test_jobs_refresh_with_keep_completed(clean_jobs, subject, experiment): + """Test that jobs.refresh() works with keep_completed=True. + + Regression test for https://github.com/datajoint/datajoint-python/issues/1379 + """ + # Clear experiment data and jobs + experiment.delete() + experiment.jobs.delete() + + with dj.config.override(jobs={"keep_completed": True, "add_job_metadata": True}): + # Refresh should create pending jobs without semantic matching error + experiment.jobs.refresh() + pending_before = len(experiment.jobs.pending) + assert pending_before > 0, "No pending jobs created" + + # Manually reserve and complete a job + key = experiment.jobs.pending.keys(limit=1)[0] + experiment.jobs.reserve(key) + experiment.jobs.complete(key) + + # Job should now be completed + assert len(experiment.jobs.completed) == 1, "Job not marked as completed" + + # Calling refresh again should not raise semantic matching error + experiment.jobs.refresh() # This was failing before the fix