diff --git a/cms/db/submission.py b/cms/db/submission.py index 10647e1d11..f2997cb5c9 100644 --- a/cms/db/submission.py +++ b/cms/db/submission.py @@ -766,6 +766,9 @@ class Evaluation(Base): nullable=False, default=[]) + # Admin-facing output from the grader. + admin_text: str | None = Column(String, nullable=True, default=None) + # Evaluation's time and wall-clock time, in seconds. execution_time: float | None = Column( Float, diff --git a/cms/grading/Job.py b/cms/grading/Job.py index 3b08094d02..87493c0f1e 100644 --- a/cms/grading/Job.py +++ b/cms/grading/Job.py @@ -93,6 +93,7 @@ def __init__( info: str | None = None, success: bool | None = None, text: list[str] | None = None, + admin_text: str | None = None, files: dict[str, File] | None = None, managers: dict[str, Manager] | None = None, executables: dict[str, Executable] | None = None, @@ -121,6 +122,8 @@ def __init__( to be presented to the user. The first item is a string, potentially with %-escaping; the following items are the values to be %-formatted into the first. + admin_text: description of the outcome of the job, + to be shown to admins. files: files submitted by the user. managers: managers provided by the admins. executables: executables created in the compilation. @@ -155,6 +158,7 @@ def __init__( self.success = success self.text = text + self.admin_text = admin_text self.files = files self.managers = managers @@ -178,6 +182,7 @@ def export_to_dict(self) -> dict: 'info': self.info, 'success': self.success, 'text': self.text, + 'admin_text': self.admin_text, 'files': dict((k, v.digest) for k, v in self.files.items()), 'managers': dict((k, v.digest) @@ -316,6 +321,7 @@ def __init__( compilation_success: bool | None = None, executables: dict[str, Executable] | None = None, text: list[str] | None = None, + admin_text: str | None = None, plus: dict | None = None, ): """Initialization. @@ -331,7 +337,7 @@ def __init__( Job.__init__(self, operation, task_type, task_type_parameters, language, multithreaded_sandbox, archive_sandbox, shard, keep_sandbox, sandboxes, sandbox_digests, info, success, - text, files, managers, executables) + text, admin_text, files, managers, executables) self.compilation_success = compilation_success self.plus = plus @@ -537,6 +543,7 @@ def __init__( success: bool | None = None, outcome: str | None = None, text: list[str] | None = None, + admin_text: str | None = None, user_output: str | None = None, plus: dict | None = None, only_execution: bool | None = False, @@ -567,7 +574,7 @@ def __init__( Job.__init__(self, operation, task_type, task_type_parameters, language, multithreaded_sandbox, archive_sandbox, shard, keep_sandbox, sandboxes, sandbox_digests, info, success, - text, files, managers, executables) + text, admin_text, files, managers, executables) self.input = input self.output = output self.time_limit = time_limit @@ -653,6 +660,7 @@ def to_submission(self, sr: SubmissionResult): sr.evaluations += [Evaluation( text=self.text, + admin_text=self.admin_text, outcome=self.outcome, execution_time=self.plus.get('execution_time'), execution_wall_clock_time=self.plus.get( diff --git a/cms/grading/scoretypes/abc.py b/cms/grading/scoretypes/abc.py index f36542606e..669cade14f 100644 --- a/cms/grading/scoretypes/abc.py +++ b/cms/grading/scoretypes/abc.py @@ -144,8 +144,8 @@ def get_html_details( translation=translation, gettext=_, ngettext=n_) except Exception: - logger.error("Found an invalid score details string. " - "Try invalidating scores.") + logger.exception("Found an invalid score details string. " + "Try invalidating scores.") return _("Score details temporarily unavailable.") @abstractmethod diff --git a/cms/grading/steps/trusted.py b/cms/grading/steps/trusted.py index 7c6513876e..dce3d297ab 100644 --- a/cms/grading/steps/trusted.py +++ b/cms/grading/steps/trusted.py @@ -77,13 +77,14 @@ def _sanitize_message(string: str) -> str: return string.replace('%', '%%') -def extract_outcome_and_text(sandbox: Sandbox) -> tuple[float, list[str]]: +def extract_outcome_and_text(sandbox: Sandbox) -> tuple[float, list[str], str | None]: """Extract the outcome and the text from the a standard manager output. sandbox: the sandbox whose last execution was a manager writing a standard manager output. - return: outcome and text. + return: outcome, contestant-facing text and admin-facing text + (not translated). raise (ValueError): if cannot decode the data. raise (FileNotFoundError): if any of the sandbox stdout or stderr file @@ -108,6 +109,23 @@ def extract_outcome_and_text(sandbox: Sandbox) -> tuple[float, list[str]]: logger.error("Manager stderr (text) is malformed. %r", error) raise error + # Parse special commands + admin_text = None + for line in stderr_file.readlines(): + line = line.strip() + if not line: + continue + + PREFIX = "ADMIN_MESSAGE:" + if line.startswith(PREFIX): + line = _sanitize_message(line[len(PREFIX):].strip()) + if admin_text is not None: + admin_text = admin_text + " " + line + else: + admin_text = line + else: + logger.warning(f"Unknown special manager command `{line}`") + try: outcome = float(outcome) except ValueError: @@ -125,7 +143,7 @@ def extract_outcome_and_text(sandbox: Sandbox) -> tuple[float, list[str]]: logger.warning("Manager asked to translate text, but string " "'%s' is not recognized." % remaining) - return outcome, [text] + return outcome, [text], admin_text def trusted_step( @@ -196,7 +214,7 @@ def checker_step( correct_output_digest: str, output_filename: str, extra_args: list[str] | None = None -) -> tuple[bool, float | None, list[str] | None]: +) -> tuple[bool, float | None, list[str] | None, str | None]: """Run the explicit checker given by the admins sandbox: the sandbox to run the checker in; should already @@ -213,7 +231,8 @@ def checker_step( extra_args: extra arguments to pass to the checker. return: success (true if the checker was able to check the solution - successfully), outcome and text (both None if success is False). + successfully), outcome, text and admin_text (all None if success + is False). """ # Check that the file we are going to inject in the sandbox are not already @@ -224,12 +243,12 @@ def checker_step( if sandbox.file_exists(filename): logger.error("File %s already in the sandbox for the checker.", filename) - return False, None, None + return False, None, None, None # Copy the checker in the sandbox, after making sure it was provided. if checker_digest is None: logger.error("Configuration error: missing checker in task managers.") - return False, None, None + return False, None, None, None sandbox.create_file_from_storage(CHECKER_FILENAME, checker_digest, executable=True) @@ -247,17 +266,17 @@ def checker_step( if not box_success or not success: logger.error("Sandbox failed during checker step. " "See previous logs for the reason.") - return False, None, None + return False, None, None, None # Extract outcome and text assuming a standard manager output. try: - outcome, text = extract_outcome_and_text(sandbox) + outcome, text, admin_text = extract_outcome_and_text(sandbox) except ValueError as e: logger.error("Invalid output from checker: %s", e) - return False, None, None + return False, None, None, None except FileNotFoundError as e: # This should not happen, as the redirect is handled by the sandbox. logger.error("Missing stdout or stderr file from checker: %s", e) - return False, None, None + return False, None, None, None - return True, outcome, text + return True, outcome, text, admin_text diff --git a/cms/grading/steps/whitediff.py b/cms/grading/steps/whitediff.py index 395bcf8420..f5b7d39e13 100644 --- a/cms/grading/steps/whitediff.py +++ b/cms/grading/steps/whitediff.py @@ -72,7 +72,7 @@ def _white_diff_canonicalize(string: bytes) -> bytes: return string -def _white_diff(output: typing.BinaryIO, res: typing.BinaryIO) -> bool: +def _white_diff(output: typing.BinaryIO, res: typing.BinaryIO) -> tuple[bool, str | None]: """Compare the two output files. Two files are equal if for every integer i, line i of first file is equal to line i of second file. Two lines are equal if they differ only by number or type of @@ -89,20 +89,25 @@ def _white_diff(output: typing.BinaryIO, res: typing.BinaryIO) -> bool: """ + line = 0 + while True: lout = output.readline() lres = res.readline() + line += 1 # Both files finished: comparison succeded if len(lres) == 0 and len(lout) == 0: - return True + return True, None # Only one file finished: ok if the other contains only blanks elif len(lres) == 0 or len(lout) == 0: lout = lout.strip(b''.join(_WHITES)) lres = lres.strip(b''.join(_WHITES)) - if len(lout) > 0 or len(lres) > 0: - return False + if len(lout) > 0: + return False, "Contestant output too long" + if len(lres) > 0: + return False, "Contestant output too short" # Both file still have lines to go: ok if they agree except # for the number of whitespaces @@ -110,12 +115,19 @@ def _white_diff(output: typing.BinaryIO, res: typing.BinaryIO) -> bool: lout = _white_diff_canonicalize(lout) lres = _white_diff_canonicalize(lres) if lout != lres: - return False + LENGTH_LIMIT = 100 + if len(lout) > LENGTH_LIMIT: + lout = lout[:LENGTH_LIMIT] + b"..." + if len(lres) > LENGTH_LIMIT: + lres = lres[:LENGTH_LIMIT] + b"..." + lout = lout.decode("utf-8", errors='backslashreplace') + lres = lres.decode("utf-8", errors='backslashreplace') + return False, f"Expected `{lres}`, found `{lout}` on line {line}" def white_diff_fobj_step( output_fobj: typing.BinaryIO, correct_output_fobj: typing.BinaryIO -) -> tuple[float, list[str]]: +) -> tuple[float, list[str], str | None]: """Compare user output and correct output with a simple diff. It gives an outcome 1.0 if the output and the reference output are @@ -129,15 +141,16 @@ def white_diff_fobj_step( return: the outcome as above and a description text. """ - if _white_diff(output_fobj, correct_output_fobj): - return 1.0, [EVALUATION_MESSAGES.get("success").message] + correct, admin_text = _white_diff(output_fobj, correct_output_fobj) + if correct: + return 1.0, [EVALUATION_MESSAGES.get("success").message], admin_text else: - return 0.0, [EVALUATION_MESSAGES.get("wrong").message] + return 0.0, [EVALUATION_MESSAGES.get("wrong").message], admin_text def white_diff_step( sandbox: Sandbox, output_filename: str, correct_output_filename: str -) -> tuple[float, list[str]]: +) -> tuple[float, list[str], str | None]: """Compare user output and correct output with a simple diff. It gives an outcome 1.0 if the output and the reference output are @@ -157,4 +170,4 @@ def white_diff_step( return white_diff_fobj_step(out_file, res_file) else: return 0.0, [ - EVALUATION_MESSAGES.get("nooutput").message, output_filename] + EVALUATION_MESSAGES.get("nooutput").message, output_filename], None diff --git a/cms/grading/tasktypes/Batch.py b/cms/grading/tasktypes/Batch.py index d23ab04e51..5479fe6a92 100644 --- a/cms/grading/tasktypes/Batch.py +++ b/cms/grading/tasktypes/Batch.py @@ -364,10 +364,12 @@ def _execution_step(self, job, file_cacher): return outcome, text, output_file_params, stats, box_success, sandbox def _evaluate_step(self, job, file_cacher, output_file_params, outcome, text, stats, box_success, sandbox, extra_args): + admin_text = None + if box_success: assert (output_file_params is None) == (outcome is not None) if output_file_params is not None: - box_success, outcome, text = eval_output( + box_success, outcome, text, admin_text = eval_output( file_cacher, job, self.CHECKER_CODENAME if self._uses_checker() else None, @@ -378,6 +380,7 @@ def _evaluate_step(self, job, file_cacher, output_file_params, outcome, text, st job.outcome = str(outcome) if outcome is not None else None job.text = text job.plus = stats + job.admin_text = admin_text if sandbox is not None: delete_sandbox(sandbox, job) diff --git a/cms/grading/tasktypes/Communication.py b/cms/grading/tasktypes/Communication.py index 60f53a993b..8085ed8297 100644 --- a/cms/grading/tasktypes/Communication.py +++ b/cms/grading/tasktypes/Communication.py @@ -396,6 +396,7 @@ def evaluate(self, job, file_cacher): and box_success_mgr and evaluation_success_mgr outcome = None text = None + admin_text = None # If at least one sandbox had problems, or the manager did not # terminate correctly, we report an error (and no need for user stats). @@ -415,7 +416,7 @@ def evaluate(self, job, file_cacher): # Otherwise, we use the manager to obtain the outcome. else: - outcome, text = extract_outcome_and_text(sandbox_mgr) + outcome, text, admin_text = extract_outcome_and_text(sandbox_mgr) # If asked so, save the output file with additional information, # provided that it exists. @@ -433,6 +434,7 @@ def evaluate(self, job, file_cacher): job.outcome = "%s" % outcome if outcome is not None else None job.text = text job.plus = stats_user + job.admin_text = admin_text delete_sandbox(sandbox_mgr, job) for s in sandbox_user: diff --git a/cms/grading/tasktypes/OutputOnly.py b/cms/grading/tasktypes/OutputOnly.py index 7a5e2e3e00..f5e6a7683e 100644 --- a/cms/grading/tasktypes/OutputOnly.py +++ b/cms/grading/tasktypes/OutputOnly.py @@ -124,7 +124,7 @@ def evaluate(self, job, file_cacher): return # First and only step: eval the user output. - box_success, outcome, text = eval_output( + box_success, outcome, text, admin_text = eval_output( file_cacher, job, OutputOnly.CHECKER_CODENAME if self._uses_checker() else None, user_output_digest=job.files[user_output_filename].digest) @@ -133,5 +133,6 @@ def evaluate(self, job, file_cacher): job.success = box_success job.outcome = str(outcome) if outcome is not None else None job.text = text + job.admin_text = admin_text # There is no actual evaluation, so no statistics. job.plus = {} if box_success else None diff --git a/cms/grading/tasktypes/TwoSteps.py b/cms/grading/tasktypes/TwoSteps.py index fbe0ee5ce4..805d7112de 100644 --- a/cms/grading/tasktypes/TwoSteps.py +++ b/cms/grading/tasktypes/TwoSteps.py @@ -295,6 +295,7 @@ def evaluate(self, job, file_cacher): outcome = None text = None + admin_text = None # Error in the sandbox: nothing to do! if not box_success: @@ -333,7 +334,7 @@ def evaluate(self, job, file_cacher): # Otherwise evaluate the output file. else: - box_success, outcome, text = eval_output( + box_success, outcome, text, admin_text = eval_output( file_cacher, job, TwoSteps.CHECKER_CODENAME if self._uses_checker() else None, @@ -344,6 +345,7 @@ def evaluate(self, job, file_cacher): job.success = box_success job.outcome = str(outcome) if outcome is not None else None job.text = text + job.admin_text = admin_text job.plus = stats delete_sandbox(first_sandbox, job) diff --git a/cms/grading/tasktypes/util.py b/cms/grading/tasktypes/util.py index 609d7c5ac6..c4efb5f270 100644 --- a/cms/grading/tasktypes/util.py +++ b/cms/grading/tasktypes/util.py @@ -221,7 +221,7 @@ def eval_output( user_output_digest: str | None = None, user_output_filename: str = "", extra_args: list[str] | None = None -) -> tuple[bool, float | None, list[str] | None]: +) -> tuple[bool, float | None, list[str] | None, str | None]: """Evaluate ("check") a user output using a white diff or a checker. file_cacher: file cacher to use to get files. @@ -237,8 +237,8 @@ def eval_output( extra_args: additional arguments to pass to the checker return: tuple of success (true if the checker was - able to check the solution successfully), outcome and text (both None - if success is False). + able to check the solution successfully), outcome, text and admin_text + (both None if success is False). """ if (user_output_path is None) == (user_output_digest is None): @@ -252,11 +252,11 @@ def eval_output( if not os.path.exists(user_output_path) \ or os.path.islink(user_output_path): return True, 0.0, [EVALUATION_MESSAGES.get("nooutput").message, - user_output_filename] + user_output_filename], None if checker_codename is not None: if not check_manager_present(job, checker_codename): - return False, None, None + return False, None, None, None # Create a brand-new sandbox just for checking. sandbox = create_sandbox(file_cacher, name="check") @@ -275,12 +275,12 @@ def eval_output( checker_digest = job.managers[checker_codename].digest \ if checker_codename in job.managers else None - success, outcome, text = checker_step( + success, outcome, text, admin_text = checker_step( sandbox, checker_digest, job.input, job.output, EVAL_USER_OUTPUT_FILENAME, extra_args) delete_sandbox(sandbox, job, success) - return success, outcome, text + return success, outcome, text, admin_text else: if user_output_path is not None: @@ -289,6 +289,6 @@ def eval_output( user_output_fobj = file_cacher.get_file(user_output_digest) with user_output_fobj: with file_cacher.get_file(job.output) as correct_output_fobj: - outcome, text = white_diff_fobj_step( + outcome, text, admin_text = white_diff_fobj_step( user_output_fobj, correct_output_fobj) - return True, outcome, text + return True, outcome, text, admin_text diff --git a/cms/server/admin/templates/submission.html b/cms/server/admin/templates/submission.html index 940854288d..727e043671 100644 --- a/cms/server/admin/templates/submission.html +++ b/cms/server/admin/templates/submission.html @@ -261,6 +261,7 @@

Evaluation (as seen by the a Outcome Visible Details + Admin Details Shard Resources Sandbox @@ -277,6 +278,13 @@

Evaluation (as seen by the a {{ ev.text|format_status_text }} + + {% if ev.admin_text is not none %} + {{ ev.admin_text }} + {% else %} + N/A + {% endif %} + {{ ev.evaluation_shard }} {% if ev.execution_time is not none %} diff --git a/cmscontrib/updaters/update_from_1.5.sql b/cmscontrib/updaters/update_from_1.5.sql index f35188a064..1849967d62 100644 --- a/cmscontrib/updaters/update_from_1.5.sql +++ b/cmscontrib/updaters/update_from_1.5.sql @@ -48,4 +48,7 @@ ALTER TABLE public.tasks ADD COLUMN allowed_languages varchar[]; -- https://github.com/cms-dev/cms/pull/1583 DROP TABLE public.printjobs; +-- https://github.com/cms-dev/cms/pull/1642 +ALTER TABLE evaluations ADD COLUMN admin_text VARCHAR; + COMMIT; diff --git a/cmstestsuite/unit_tests/grading/steps/trusted_test.py b/cmstestsuite/unit_tests/grading/steps/trusted_test.py index 82cf6f12e1..f6921b4b6b 100755 --- a/cmstestsuite/unit_tests/grading/steps/trusted_test.py +++ b/cmstestsuite/unit_tests/grading/steps/trusted_test.py @@ -47,35 +47,43 @@ def setUp(self): def test_success(self): self.sandbox.fake_file("o", b"0.45\n") self.sandbox.fake_file("e", "你好.\n".encode("utf-8")) - outcome, text = extract_outcome_and_text(self.sandbox) + outcome, text, admin_text = extract_outcome_and_text(self.sandbox) self.assertEqual(outcome, 0.45) self.assertEqual(text, ["你好."]) + def test_admin_message(self): + self.sandbox.fake_file("o", b"0.45\n") + self.sandbox.fake_file("e", b"Text to return.\nADMIN_MESSAGE: admin") + outcome, text, admin_text = extract_outcome_and_text(self.sandbox) + self.assertEqual(outcome, 0.45) + self.assertEqual(text, ["Text to return."]) + self.assertEqual(admin_text, "admin") + def test_following_lines_ignored(self): self.sandbox.fake_file("o", b"0.45\nNothing\n") self.sandbox.fake_file("e", b"Text to return.\nto see here") - outcome, text = extract_outcome_and_text(self.sandbox) + outcome, text, admin_text = extract_outcome_and_text(self.sandbox) self.assertEqual(outcome, 0.45) self.assertEqual(text, ["Text to return."]) def test_works_without_newlines(self): self.sandbox.fake_file("o", b"0.45") self.sandbox.fake_file("e", b"Text to return.") - outcome, text = extract_outcome_and_text(self.sandbox) + outcome, text, admin_text = extract_outcome_and_text(self.sandbox) self.assertEqual(outcome, 0.45) self.assertEqual(text, ["Text to return."]) def test_text_is_stripped(self): self.sandbox.fake_file("o", b" 0.45\t \nignored") self.sandbox.fake_file("e", b"\t Text to return.\r\n") - outcome, text = extract_outcome_and_text(self.sandbox) + outcome, text, admin_text = extract_outcome_and_text(self.sandbox) self.assertEqual(outcome, 0.45) self.assertEqual(text, ["Text to return."]) def test_text_is_translated(self): self.sandbox.fake_file("o", b"0.45\n") self.sandbox.fake_file("e", b"translate:success\n") - outcome, text = extract_outcome_and_text(self.sandbox) + outcome, text, admin_text = extract_outcome_and_text(self.sandbox) self.assertEqual(outcome, 0.45) self.assertEqual(text, ["Output is correct"]) @@ -260,7 +268,7 @@ def test_success(self): ret = checker_step(self.sandbox, "c_dig", "i_dig", "co_dig", "o") - self.assertEqual(ret, (True, 0.123, ["Text."])) + self.assertEqual(ret, (True, 0.123, ["Text."], None)) self.file_cacher.get_file_to_fobj.assert_has_calls([ call("c_dig", ANY), call("i_dig", ANY), @@ -278,7 +286,7 @@ def test_sandbox_failure(self): ret = checker_step(self.sandbox, "c_dig", "i_dig", "co_dig", "o") - self.assertEqual(ret, (False, None, None)) + self.assertEqual(ret, (False, None, None, None)) self.assertLoggedError() def test_checker_failure(self): @@ -288,28 +296,28 @@ def test_checker_failure(self): ret = checker_step(self.sandbox, "c_dig", "i_dig", "co_dig", "o") - self.assertEqual(ret, (False, None, None)) + self.assertEqual(ret, (False, None, None, None)) self.assertLoggedError() def test_missing_checker(self): ret = checker_step(self.sandbox, None, "i_dig", "co_dig", "o") self.mock_trusted_step.assert_not_called() - self.assertEqual(ret, (False, None, None)) + self.assertEqual(ret, (False, None, None, None)) self.assertLoggedError() def test_checker_already_in_sandbox(self): self.sandbox.fake_file(trusted.CHECKER_FILENAME, b"something") ret = checker_step(self.sandbox, "c_dig", "i_dig", "co_dig", "o") - self.assertEqual(ret, (False, None, None)) + self.assertEqual(ret, (False, None, None, None)) self.assertLoggedError() def test_input_already_in_sandbox(self): self.sandbox.fake_file(trusted.CHECKER_INPUT_FILENAME, b"something") ret = checker_step(self.sandbox, "c_dig", "i_dig", "co_dig", "o") - self.assertEqual(ret, (False, None, None)) + self.assertEqual(ret, (False, None, None, None)) self.assertLoggedError() def test_correct_output_already_in_sandbox(self): @@ -317,7 +325,7 @@ def test_correct_output_already_in_sandbox(self): b"something") ret = checker_step(self.sandbox, "c_dig", "i_dig", "co_dig", "o") - self.assertEqual(ret, (False, None, None)) + self.assertEqual(ret, (False, None, None, None)) self.assertLoggedError() def test_invalid_checker_outcome(self): @@ -326,7 +334,7 @@ def test_invalid_checker_outcome(self): ret = checker_step(self.sandbox, "c_dig", "i_dig", "co_dig", "o") - self.assertEqual(ret, (False, None, None)) + self.assertEqual(ret, (False, None, None, None)) self.assertLoggedError() def test_invalid_checker_text(self): @@ -335,7 +343,7 @@ def test_invalid_checker_text(self): ret = checker_step(self.sandbox, "c_dig", "i_dig", "co_dig", "o") - self.assertEqual(ret, (False, None, None)) + self.assertEqual(ret, (False, None, None, None)) self.assertLoggedError() def test_missing_checker_outcome(self): @@ -344,7 +352,7 @@ def test_missing_checker_outcome(self): ret = checker_step(self.sandbox, "c_dig", "i_dig", "co_dig", "o") - self.assertEqual(ret, (False, None, None)) + self.assertEqual(ret, (False, None, None, None)) self.assertLoggedError() def test_missing_checker_text(self): @@ -353,7 +361,7 @@ def test_missing_checker_text(self): ret = checker_step(self.sandbox, "c_dig", "i_dig", "co_dig", "o") - self.assertEqual(ret, (False, None, None)) + self.assertEqual(ret, (False, None, None, None)) self.assertLoggedError() diff --git a/cmstestsuite/unit_tests/grading/steps/whitediff_test.py b/cmstestsuite/unit_tests/grading/steps/whitediff_test.py index 595c2e4a9b..22c54e6e97 100755 --- a/cmstestsuite/unit_tests/grading/steps/whitediff_test.py +++ b/cmstestsuite/unit_tests/grading/steps/whitediff_test.py @@ -31,7 +31,7 @@ class TestWhiteDiff(unittest.TestCase): @staticmethod def _diff(s1, s2): return _white_diff( - BytesIO(s1.encode("utf-8")), BytesIO(s2.encode("utf-8"))) + BytesIO(s1.encode("utf-8")), BytesIO(s2.encode("utf-8")))[0] def test_no_diff_one_token(self): self.assertTrue(self._diff("", "")) @@ -67,6 +67,11 @@ def test_diff_wrong_line(self): self.assertFalse(self._diff("1 2", "1\n2")) self.assertFalse(self._diff("1\n\n2", "1\n2")) + def test_diff_wrong_long_line(self): + line1 = "1" * 1000 + line2 = line1 + "0" + self.assertFalse(self._diff(line1, line2)) + if __name__ == "__main__": unittest.main() diff --git a/cmstestsuite/unit_tests/grading/tasktypes/BatchTest.py b/cmstestsuite/unit_tests/grading/tasktypes/BatchTest.py index f56935111a..51fafec662 100755 --- a/cmstestsuite/unit_tests/grading/tasktypes/BatchTest.py +++ b/cmstestsuite/unit_tests/grading/tasktypes/BatchTest.py @@ -305,7 +305,7 @@ def prepare(self, parameters, executables): tt = Batch(parameters) job = self.job(executables) self.evaluation_step.return_value = (True, True, STATS_OK) - self.eval_output.return_value = (True, OUTCOME, TEXT) + self.eval_output.return_value = (True, OUTCOME, TEXT, None) return tt, job def assertResultsInJob(self, job): @@ -325,7 +325,7 @@ def assertResultsInJob(self, job): else: # User submission terminated correctly, output is evaluated. _, _, stats = self.evaluation_step.return_value - success, outcome, text = self.eval_output.return_value + success, outcome, text, admin_text = self.eval_output.return_value if isinstance(outcome, float): outcome = str(outcome) @@ -416,7 +416,7 @@ def test_stdio_diff_evaluation_step_sandbox_failure_(self): def test_stdio_diff_eval_output_failure_(self): tt, job = self.prepare(["alone", ["", ""], "diff"], {"foo": EXE_FOO}) - self.eval_output.return_value = (False, None, None) + self.eval_output.return_value = (False, None, None, None) sandbox = self.expect_sandbox() tt.evaluate(job, self.file_cacher) diff --git a/cmstestsuite/unit_tests/grading/tasktypes/CommunicationTest.py b/cmstestsuite/unit_tests/grading/tasktypes/CommunicationTest.py index bdd298dd0c..cf2ea1ff41 100755 --- a/cmstestsuite/unit_tests/grading/tasktypes/CommunicationTest.py +++ b/cmstestsuite/unit_tests/grading/tasktypes/CommunicationTest.py @@ -330,7 +330,7 @@ def prepare(self, parameters, executables, managers): tt = Communication(parameters) job = self.job(executables, managers) self.evaluation_step_after_run.return_value = (True, True, STATS_OK) - self.extract_outcome_and_text.return_value = (OUTCOME, TEXT) + self.extract_outcome_and_text.return_value = (OUTCOME, TEXT, None) return tt, job def assertResultsInJob(self, job, success, outcome, text, stats): diff --git a/cmstestsuite/unit_tests/grading/tasktypes/OutputOnlyTest.py b/cmstestsuite/unit_tests/grading/tasktypes/OutputOnlyTest.py index 45172cce16..f4d4d0add4 100755 --- a/cmstestsuite/unit_tests/grading/tasktypes/OutputOnlyTest.py +++ b/cmstestsuite/unit_tests/grading/tasktypes/OutputOnlyTest.py @@ -58,7 +58,7 @@ def job(files): def prepare(self, parameters, files): tt = OutputOnly(parameters) job = self.job(files) - self.eval_output.return_value = (True, OUTCOME, TEXT) + self.eval_output.return_value = (True, OUTCOME, TEXT, None) return tt, job def assertResultsInJob(self, job, success, outcome, text, stats): @@ -95,7 +95,7 @@ def test_diff_failure(self): "output_001.txt": FILE_001, "output_023.txt": FILE_023 }) - self.eval_output.return_value = False, None, None + self.eval_output.return_value = False, None, None, None tt.evaluate(job, self.file_cacher)