diff --git a/renderer.php b/renderer.php index a0b9d071..4f952d82 100644 --- a/renderer.php +++ b/renderer.php @@ -163,8 +163,11 @@ public function part_formulation_and_controls( $feedback .= $this->part_correct_response($part); } - // Put all feedback into a
with the appropriate CSS class and append it to the output. - $output .= html_writer::nonempty_tag('div', $feedback, ['class' => 'formulaspartoutcome outcome']); + // If the current response is a real submission (or identical to one), we put all feedback into a + //
with the appropriate CSS class and append it to the output. + if ($this->response_is_same_as_submitted($qa, $part)) { + $output .= html_writer::nonempty_tag('div', $feedback, ['class' => 'formulaspartoutcome outcome']); + } return html_writer::tag('div', $output, ['class' => 'formulaspart']); } @@ -200,7 +203,9 @@ public function get_part_feedback_class_and_symbol( $result->feedbacksymbol = ''; $result->feedbackclass = ''; // ... unless correctness is requested in the display options. - if ($options->correctness) { + // Note that no feedback should be given, if the response has been modified since the last submission, + // i. e. it is just a response that was saved during page navigation. + if ($this->response_is_same_as_submitted($qa, $part) && $options->correctness) { $result->feedbacksymbol = $this->feedback_image($result->fraction); $result->feedbackclass = $this->feedback_class($result->fraction); } @@ -859,6 +864,57 @@ public function part_correct_response($part) { ); } + /** + * Check whether the last response of a question attempt is the same as the last submitted response, i. e. it + * was either submitted (e. g. using the "Check" button) or it was saved during page navigation in a quiz but + * still contains the same answers as the ones from the last regular submission. + * + * @param question_attempt $qa + * @param formulas_part|null $part + * @return bool + */ + protected function response_is_same_as_submitted(question_attempt $qa, formulas_part|null $part = null): bool { + // If the last step contains the behaviour var 'submit', it was itself a submitted response. + // For the deferredfeedback behaviour, the step will contain 'finish' instead of 'submit'. + $laststep = $qa->get_last_step(); + if ($laststep->has_behaviour_var('submit') || $laststep->has_behaviour_var('finish')) { + return true; + } + + // Otherwise, we try to fetch the step containing the last submitted response. + $lastsubmitted = $qa->get_last_step_with_behaviour_var('submit'); + $lastsubmitteddata = $lastsubmitted->get_qt_data(); + + // If there is no data, then no response has ever been submitted. + if (empty($lastsubmitteddata)) { + return false; + } + + // If we have a part, we compare the last step's data to the one from the last submitted response, + // but only for the fields of the relevant part. + $lastdata = $laststep->get_qt_data(); + if ($part !== null) { + return $part->is_same_response($lastsubmitteddata, $lastdata); + } + + // If we do not have a part, we compare the reponse for the entire question. + /** @var qtype_formulas_question $question */ + $question = $qa->get_question(); + + return $question->is_same_response($lastsubmitteddata, $lastdata); + } + + #[\Override] + public function feedback(question_attempt $qa, question_display_options $options) { + // We should not give feedback if the response is not properly submitted, but rather just saved + // during navigation through the quiz. + if (!$this->response_is_same_as_submitted($qa)) { + return ''; + } + + return parent::feedback($qa, $options); + } + /** * Generate a brief statement of how many sub-parts of this question the * student got right. @@ -979,9 +1035,9 @@ protected function part_general_feedback(question_attempt $qa, question_display_ $gradingdetailsdiv = $renderer->render_adaptive_marks($details, $options); $state = $details->state; } - // If the question is in a state that does not yet allow to give a feedback, - // we return an empty string. - if (empty($state->get_feedback_class())) { + // If the question is in a state that does not yet allow to give a feedback + // or if the response is not the last one to be checked, we return an empty string. + if (!$this->response_is_same_as_submitted($qa, $part) || empty($state->get_feedback_class())) { return ''; } diff --git a/tests/behat/adaptivenoleak.feature b/tests/behat/adaptivenoleak.feature new file mode 100644 index 00000000..20ec263b --- /dev/null +++ b/tests/behat/adaptivenoleak.feature @@ -0,0 +1,110 @@ +@qtype @qtype_formulas +Feature: Make sure we do not leak information in adaptive mode + + Background: + Given the following "users" exist: + | username | + | student | + And the following "courses" exist: + | fullname | shortname | category | + | Course 1 | C1 | 0 | + And the following "course enrolments" exist: + | user | course | role | + | student | C1 | student | + And the following "question categories" exist: + | contextlevel | reference | name | + | Course | C1 | Test questions | + And the following "questions" exist: + | questioncategory | qtype | name | template | + | Test questions | formulas | threeparts | testthreeparts | + And the following "activities" exist: + | activity | name | course | idnumber | preferredbehaviour | + | quiz | Quiz 1 | C1 | quiz1 | adaptive | + And quiz "Quiz 1" contains the following questions: + | question | page | + | threeparts | 1 | + And I log in as "student" + And I am on "Course 1" course homepage + And I follow "Quiz 1" + And I press "Attempt quiz" + + Scenario: Try to leak number of correct parts via navigation alone + When I set the field "Answer for part 1" to "5" + And I set the field "Answer for part 2" to "6" + And I set the field "Answer for part 3" to "7" + And I press "Finish attempt" + And I press "Return to attempt" + Then ".formulaspartfeedback-0" "css_element" should not exist + And ".formulaspartfeedback-1" "css_element" should not exist + And ".formulaspartfeedback-2" "css_element" should not exist + And ".numpartscorrect" "css_element" should not exist + And ".fa-circle-check" "css_element" should not exist + + Scenario: Try to leak number of correct parts via partial submission and navigation + When I set the field "Answer for part 1" to "5" + And I press "Check" + Then I should see "Marks for this submission" in the ".formulaspartfeedback-0 .gradingdetails" "css_element" + And I should see "Part 1 correct feedback." + And I should see "You have correctly answered 1 part of this question." + And ".fa-circle-check" "css_element" should exist + When I set the field "Answer for part 2" to "6" + And I press "Finish attempt" + And I press "Return to attempt" + Then I should see "Marks for this submission" in the ".formulaspartfeedback-0 .gradingdetails" "css_element" + And I should see "Part 1 correct feedback." + And ".formulaspartfeedback-1" "css_element" should not exist + And ".numpartscorrect" "css_element" should not exist + + Scenario: Part feedback is not shown if answer has been modified (from correct to wrong) since last check + When I set the field "Answer for part 1" to "5" + And I press "Check" + Then I should see "Marks for this submission" in the ".formulaspartfeedback-0 .gradingdetails" "css_element" + And I should see "Part 1 correct feedback." + And I should see "You have correctly answered 1 part of this question." + And ".fa-circle-check" "css_element" should exist + When I set the field "Answer for part 1" to "6" + And I press "Finish attempt" + And I press "Return to attempt" + Then ".formulaspartfeedback-0" "css_element" should not exist + And ".gradingdetails" "css_element" should not exist + And ".numpartscorrect" "css_element" should not exist + And ".fa-circle-xmark" "css_element" should not exist + And ".fa-circle-check" "css_element" should not exist + + Scenario: Part feedback is not shown if answer has been modified (from wrong to correct) since last check + When I set the field "Answer for part 1" to "6" + And I press "Check" + Then I should see "Marks for this submission: 0.00" in the ".formulaspartfeedback-0 .gradingdetails" "css_element" + And I should see "Part 1 incorrect feedback." + And I should see "You have correctly answered 0 parts of this question." + And ".fa-circle-xmark" "css_element" should exist + When I set the field "Answer for part 1" to "5" + And I press "Finish attempt" + And I press "Return to attempt" + Then ".formulaspartfeedback-0" "css_element" should not exist + And ".gradingdetails" "css_element" should not exist + And ".numpartscorrect" "css_element" should not exist + And ".fa-circle-check" "css_element" should not exist + And ".fa-circle-xmark" "css_element" should not exist + + Scenario: Part feedback is shown when navigating back with an answer identical to the last one that was checked + When I set the field "Answer for part 1" to "5" + And I press "Check" + Then I should see "Marks for this submission" in the ".formulaspartfeedback-0 .gradingdetails" "css_element" + And I should see "Part 1 correct feedback." + And I should see "You have correctly answered 1 part of this question." + And ".fa-circle-check" "css_element" should exist + When I set the field "Answer for part 1" to "6" + And I press "Finish attempt" + And I press "Return to attempt" + Then ".formulaspartfeedback-0" "css_element" should not exist + And ".gradingdetails" "css_element" should not exist + And ".numpartscorrect" "css_element" should not exist + And ".fa-circle-check" "css_element" should not exist + When I set the field "Answer for part 1" to "5" + And I press "Finish attempt" + And I press "Return to attempt" + Then I should see "Marks for this submission" in the ".formulaspartfeedback-0 .gradingdetails" "css_element" + And I should see "Part 1 correct feedback." + And I should see "You have correctly answered 1 part of this question." + And ".fa-circle-check" "css_element" should exist