ChaitanyaChakkaVsk commited on
Commit
ef4994e
·
1 Parent(s): 19ff2d5

Added annotator name based progress tracking

Browse files
backend/assignment.py CHANGED
@@ -100,6 +100,21 @@ def completed_by_annotator(deduped):
100
  return result
101
 
102
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
103
  def reserved_by_video(assignment_records, completed_by_annotator_map, now, ttl_seconds, exclude_annotator=None):
104
  """Open (not-yet-completed, not-yet-expired) reservations per video,
105
  across every OTHER annotator's assignment. A reservation stops counting
 
100
  return result
101
 
102
 
103
+ def saved_responses_for_round(deduped, annotator_id, video_ids):
104
+ """{video_id: responses} for every video in `video_ids` that `annotator_id`
105
+ already has a saved annotation for - lets a returning annotator (same
106
+ name, new session or a reload) resume with prior answers pre-filled
107
+ instead of starting blank. `responses` is exactly the {events, video}
108
+ shape the frontend's annotation state already uses, since it's what was
109
+ saved from that same shape originally."""
110
+ video_id_set = set(video_ids)
111
+ return {
112
+ video_id: record["annotations"]["responses"]
113
+ for (a_id, video_id), record in deduped.items()
114
+ if a_id == annotator_id and video_id in video_id_set
115
+ }
116
+
117
+
118
  def reserved_by_video(assignment_records, completed_by_annotator_map, now, ttl_seconds, exclude_annotator=None):
119
  """Open (not-yet-completed, not-yet-expired) reservations per video,
120
  across every OTHER annotator's assignment. A reservation stops counting
backend/main.py CHANGED
@@ -553,7 +553,7 @@ def _completed_maps(fresh=False):
553
  )
554
 
555
 
556
- def _round_response(round_record, completion):
557
  response = {
558
  "annotator_id": round_record["annotator_id"],
559
  "assignment_id": round_record["assignment_id"],
@@ -566,6 +566,12 @@ def _round_response(round_record, completion):
566
  if completion:
567
  response["completion_code"] = completion["completion_code"]
568
  response["completed_at"] = completion["completed_at"]
 
 
 
 
 
 
569
  return response
570
 
571
 
@@ -625,7 +631,11 @@ def _create_round(annotator_id, annotator_raw, round_number):
625
  f"Point {annotator_id} at round {round_number}",
626
  )
627
 
628
- return _round_response(round_record, completion=None)
 
 
 
 
629
 
630
 
631
  def get_or_create_current_round(annotator_raw):
@@ -661,13 +671,16 @@ def get_or_create_current_round(annotator_raw):
661
  completion = _read_json_from_repo(_completion_path(annotator_id, pointer["assignment_id"]))
662
 
663
  if completion is None:
664
- _deduped, _skipped, _completed_by_video_map, completed_by_annotator_map = _completed_maps(fresh=True)
665
  completed_set = completed_by_annotator_map.get(annotator_id, set())
666
  all_done = len(round_record["video_ids"]) > 0 and all(
667
  video_id in completed_set for video_id in round_record["video_ids"]
668
  )
669
  if not all_done:
670
- return _round_response(round_record, completion=None)
 
 
 
671
  completion = _persist_completion(annotator_id, round_record)
672
 
673
  completed_at = assignment.parse_iso(completion["completed_at"])
 
553
  )
554
 
555
 
556
+ def _round_response(round_record, completion, saved_responses=None):
557
  response = {
558
  "annotator_id": round_record["annotator_id"],
559
  "assignment_id": round_record["assignment_id"],
 
566
  if completion:
567
  response["completion_code"] = completion["completion_code"]
568
  response["completed_at"] = completion["completed_at"]
569
+ else:
570
+ # Lets a returning annotator (same name, new session or a reload)
571
+ # resume with prior answers pre-filled instead of the frontend
572
+ # starting every field blank - see EvaluationPage.jsx's initial
573
+ # state, which seeds from this.
574
+ response["saved_annotations"] = saved_responses or {}
575
  return response
576
 
577
 
 
631
  f"Point {annotator_id} at round {round_number}",
632
  )
633
 
634
+ # A fresh round's video_ids are, by construction, videos this annotator
635
+ # has never completed (build_round excludes already_completed from the
636
+ # candidate pool) - nothing to rehydrate yet, but saved_responses is
637
+ # always present in the response shape either way.
638
+ return _round_response(round_record, completion=None, saved_responses={})
639
 
640
 
641
  def get_or_create_current_round(annotator_raw):
 
671
  completion = _read_json_from_repo(_completion_path(annotator_id, pointer["assignment_id"]))
672
 
673
  if completion is None:
674
+ deduped, _skipped, _completed_by_video_map, completed_by_annotator_map = _completed_maps(fresh=True)
675
  completed_set = completed_by_annotator_map.get(annotator_id, set())
676
  all_done = len(round_record["video_ids"]) > 0 and all(
677
  video_id in completed_set for video_id in round_record["video_ids"]
678
  )
679
  if not all_done:
680
+ saved_responses = assignment.saved_responses_for_round(
681
+ deduped, annotator_id, round_record["video_ids"]
682
+ )
683
+ return _round_response(round_record, completion=None, saved_responses=saved_responses)
684
  completion = _persist_completion(annotator_id, round_record)
685
 
686
  completed_at = assignment.parse_iso(completion["completed_at"])
backend/test_assignment.py CHANGED
@@ -86,6 +86,35 @@ class DedupeLatestTests(unittest.TestCase):
86
  self.assertEqual(skipped, 4)
87
 
88
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
89
  class PickBalancedTests(unittest.TestCase):
90
  def test_prefers_lowest_coverage(self):
91
  coverage = {"v1": 4, "v2": 0, "v3": 2}
@@ -356,6 +385,30 @@ class RoundLifecycleIntegrationTests(unittest.TestCase):
356
  self.assertEqual(result["status"], "in_progress")
357
  self.assertNotIn("completion_code", result)
358
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
359
  def test_completion_code_persisted_once_at_twenty_of_twenty(self):
360
  first = self.backend_main.get_or_create_current_round("Scratch Tester")
361
  self._complete_round("Scratch Tester", first)
 
86
  self.assertEqual(skipped, 4)
87
 
88
 
89
+ class SavedResponsesForRoundTests(unittest.TestCase):
90
+ def test_returns_responses_only_for_this_annotator_and_this_rounds_videos(self):
91
+ records = [
92
+ make_record("Adi", "v1", "2026-01-01T00:00:00Z", {"annotations": {"responses": {"marker": "adi-v1"}}}),
93
+ make_record("Adi", "v2", "2026-01-01T00:00:00Z", {"annotations": {"responses": {"marker": "adi-v2"}}}),
94
+ make_record(
95
+ "youngsun", "v1", "2026-01-01T00:00:00Z", {"annotations": {"responses": {"marker": "youngsun-v1"}}}
96
+ ),
97
+ ]
98
+ deduped, _skipped = assignment.dedupe_latest(records)
99
+
100
+ result = assignment.saved_responses_for_round(deduped, "adi", ["v1", "v2", "v3"])
101
+
102
+ self.assertEqual(result, {"v1": {"marker": "adi-v1"}, "v2": {"marker": "adi-v2"}})
103
+
104
+ def test_excludes_videos_outside_the_given_round(self):
105
+ records = [
106
+ make_record("Adi", "v9", "2026-01-01T00:00:00Z", {"annotations": {"responses": {"marker": "old-round"}}})
107
+ ]
108
+ deduped, _skipped = assignment.dedupe_latest(records)
109
+
110
+ result = assignment.saved_responses_for_round(deduped, "adi", ["v1", "v2"])
111
+
112
+ self.assertEqual(result, {}, "a completion from a prior round must not leak into this round's resume state")
113
+
114
+ def test_empty_deduped_returns_empty_dict(self):
115
+ self.assertEqual(assignment.saved_responses_for_round({}, "adi", ["v1"]), {})
116
+
117
+
118
  class PickBalancedTests(unittest.TestCase):
119
  def test_prefers_lowest_coverage(self):
120
  coverage = {"v1": 4, "v2": 0, "v3": 2}
 
385
  self.assertEqual(result["status"], "in_progress")
386
  self.assertNotIn("completion_code", result)
387
 
388
+ def test_brand_new_round_has_no_saved_annotations(self):
389
+ result = self.backend_main.get_or_create_current_round("Scratch Tester")
390
+ self.assertEqual(result["saved_annotations"], {})
391
+
392
+ def test_resume_returns_saved_responses_for_completed_videos_only(self):
393
+ first = self.backend_main.get_or_create_current_round("Scratch Tester")
394
+ completed_ids = first["video_ids"][:3]
395
+ for video_id in completed_ids:
396
+ self.completed_records.append(
397
+ make_record(
398
+ "Scratch Tester",
399
+ video_id,
400
+ iso(datetime.now(timezone.utc)),
401
+ {"annotations": {"responses": {"video": {"tcRel": {"label": "PASS", "rationale": ""}}}}},
402
+ )
403
+ )
404
+
405
+ result = self.backend_main.get_or_create_current_round("Scratch Tester")
406
+
407
+ self.assertEqual(result["status"], "in_progress")
408
+ self.assertEqual(set(result["saved_annotations"].keys()), set(completed_ids))
409
+ for video_id in completed_ids:
410
+ self.assertEqual(result["saved_annotations"][video_id]["video"]["tcRel"]["label"], "PASS")
411
+
412
  def test_completion_code_persisted_once_at_twenty_of_twenty(self):
413
  first = self.backend_main.get_or_create_current_round("Scratch Tester")
414
  self._complete_round("Scratch Tester", first)
frontend/src/App.jsx CHANGED
@@ -156,6 +156,17 @@ export default function App() {
156
  setStage("training");
157
  };
158
 
 
 
 
 
 
 
 
 
 
 
 
159
  const handlePassTraining = () => {
160
  setTrainingPassed(true);
161
  setStage("evaluation");
@@ -241,6 +252,7 @@ export default function App() {
241
  annotatorName={annotatorName}
242
  apiBaseUrl={API_BASE_URL}
243
  onRoundFinished={fetchAssignment}
 
244
  />
245
  );
246
  }
@@ -251,6 +263,7 @@ export default function App() {
251
  onChangeAnnotatorName={setAnnotatorName}
252
  onContinue={handleContinueToTraining}
253
  apiBaseUrl={API_BASE_URL}
 
254
  />
255
  );
256
  }
 
156
  setStage("training");
157
  };
158
 
159
+ // A round can only ever exist for a name that has already passed training
160
+ // (POST /assignment is only ever called after trainingPassed is set, see
161
+ // the evaluation-stage effect above) - so GET /assignment/lookup finding
162
+ // an existing round is a reliable, read-only signal that this annotator
163
+ // already passed training in an earlier session. GuidelinePage already
164
+ // makes this exact call for its name-collision warning; this just listens
165
+ // in on that same result instead of adding a second request.
166
+ const handleGuidelineLookupResult = useCallback((data) => {
167
+ if (data?.exists) setTrainingPassed(true);
168
+ }, []);
169
+
170
  const handlePassTraining = () => {
171
  setTrainingPassed(true);
172
  setStage("evaluation");
 
252
  annotatorName={annotatorName}
253
  apiBaseUrl={API_BASE_URL}
254
  onRoundFinished={fetchAssignment}
255
+ savedAnnotations={assignmentData.saved_annotations || {}}
256
  />
257
  );
258
  }
 
263
  onChangeAnnotatorName={setAnnotatorName}
264
  onContinue={handleContinueToTraining}
265
  apiBaseUrl={API_BASE_URL}
266
+ onLookupResult={handleGuidelineLookupResult}
267
  />
268
  );
269
  }
frontend/src/pages/EvaluationPage.jsx CHANGED
@@ -8,10 +8,38 @@ import GuidelineOverlay from "./GuidelineOverlay";
8
  import axisDefinitions from "../data/axisDefinitions";
9
  import { createEmptyAnnotations } from "../lib/promptSchema";
10
 
11
- export default function EvaluationPage({ tasks, missingVideos, annotatorName, apiBaseUrl, onRoundFinished }) {
12
- const [currentIndex, setCurrentIndex] = useState(0);
13
- const [taskAnswers, setTaskAnswers] = useState({});
14
- const [savedSignatures, setSavedSignatures] = useState({});
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
15
  const [saving, setSaving] = useState(false);
16
  const [saveError, setSaveError] = useState(null);
17
  const [playbackRate, setPlaybackRate] = useState(1);
@@ -69,6 +97,10 @@ export default function EvaluationPage({ tasks, missingVideos, annotatorName, ap
69
  const hasUnsavedChanges =
70
  savedSignatures[currentTask.taskId] !== JSON.stringify(annotations);
71
 
 
 
 
 
72
  const saveAnnotations = async () => {
73
  setSaving(true);
74
  setSaveError(null);
@@ -143,7 +175,8 @@ export default function EvaluationPage({ tasks, missingVideos, annotatorName, ap
143
 
144
  <div className="eval-progress">
145
  <span>
146
- Video {currentIndex + 1} of {tasks.length} · Annotator: {annotatorName}
 
147
  </span>
148
  <button
149
  type="button"
 
8
  import axisDefinitions from "../data/axisDefinitions";
9
  import { createEmptyAnnotations } from "../lib/promptSchema";
10
 
11
+ export default function EvaluationPage({
12
+ tasks,
13
+ missingVideos,
14
+ annotatorName,
15
+ apiBaseUrl,
16
+ onRoundFinished,
17
+ savedAnnotations = {}
18
+ }) {
19
+ // Resume where a prior session (same annotator name, backend-tracked) left
20
+ // off, instead of always starting blank at video 1 - see
21
+ // App.jsx/main.py's get_or_create_current_round(), which returns
22
+ // saved_annotations for every video in this round already saved to the
23
+ // HF dataset. Lazy initializers run once on mount, matching the pattern
24
+ // already used in lib/useAnnotations.js.
25
+ const [currentIndex, setCurrentIndex] = useState(() => {
26
+ const firstUnsavedIndex = tasks.findIndex((t) => !savedAnnotations[t.taskId]);
27
+ return firstUnsavedIndex === -1 ? 0 : firstUnsavedIndex;
28
+ });
29
+ const [taskAnswers, setTaskAnswers] = useState(() => {
30
+ const initial = {};
31
+ tasks.forEach((t) => {
32
+ if (savedAnnotations[t.taskId]) initial[t.taskId] = savedAnnotations[t.taskId];
33
+ });
34
+ return initial;
35
+ });
36
+ const [savedSignatures, setSavedSignatures] = useState(() => {
37
+ const initial = {};
38
+ tasks.forEach((t) => {
39
+ if (savedAnnotations[t.taskId]) initial[t.taskId] = JSON.stringify(savedAnnotations[t.taskId]);
40
+ });
41
+ return initial;
42
+ });
43
  const [saving, setSaving] = useState(false);
44
  const [saveError, setSaveError] = useState(null);
45
  const [playbackRate, setPlaybackRate] = useState(1);
 
97
  const hasUnsavedChanges =
98
  savedSignatures[currentTask.taskId] !== JSON.stringify(annotations);
99
 
100
+ // Tracks savedSignatures (confirmed saves), not taskAnswers, so an
101
+ // in-progress draft on the current video isn't counted as "saved" yet.
102
+ const savedCount = tasks.filter((t) => savedSignatures[t.taskId] !== undefined).length;
103
+
104
  const saveAnnotations = async () => {
105
  setSaving(true);
106
  setSaveError(null);
 
175
 
176
  <div className="eval-progress">
177
  <span>
178
+ Video {currentIndex + 1} of {tasks.length} ({savedCount} saved) · Annotator:{" "}
179
+ {annotatorName}
180
  </span>
181
  <button
182
  type="button"
frontend/src/pages/GuidelinePage.jsx CHANGED
@@ -2,7 +2,13 @@
2
  import React, { useEffect, useState } from "react";
3
  import GuidelineContent from "./GuidelineContent";
4
 
5
- export default function GuidelinePage({ annotatorName, onChangeAnnotatorName, onContinue, apiBaseUrl }) {
 
 
 
 
 
 
6
  const nameIsValid = annotatorName.trim().length > 0;
7
 
8
  // Lightweight name-collision guard: there's no login, so two different
@@ -27,7 +33,13 @@ export default function GuidelinePage({ annotatorName, onChangeAnnotatorName, on
27
  fetch(`${apiBaseUrl}/assignment/lookup?annotator=${encodeURIComponent(name)}`)
28
  .then((res) => (res.ok ? res.json() : { exists: false }))
29
  .then((data) => {
30
- if (!cancelled) setCollisionInfo(data.exists ? data : null);
 
 
 
 
 
 
31
  })
32
  .catch(() => {
33
  // Fail open - this check is advisory, never block name entry on a lookup error.
@@ -39,7 +51,7 @@ export default function GuidelinePage({ annotatorName, onChangeAnnotatorName, on
39
  cancelled = true;
40
  clearTimeout(timer);
41
  };
42
- }, [annotatorName, apiBaseUrl]);
43
 
44
  const hasUnacknowledgedCollision = Boolean(collisionInfo) && !collisionAcknowledged;
45
 
@@ -55,13 +67,18 @@ export default function GuidelinePage({ annotatorName, onChangeAnnotatorName, on
55
  placeholder="Your name"
56
  />
57
  </label>
 
 
 
 
58
  {!nameIsValid && <p className="muted validation-hint">Enter your name to continue.</p>}
59
 
60
  {collisionInfo && (
61
  <div className="callout collision-warning">
62
- <span className="k">Name already in use</span>
63
  An assignment already exists for &ldquo;{annotatorName.trim()}&rdquo; (
64
- {collisionInfo.completed} of {collisionInfo.total} completed).
 
65
  <label className="confirmation-row">
66
  <input
67
  type="checkbox"
 
2
  import React, { useEffect, useState } from "react";
3
  import GuidelineContent from "./GuidelineContent";
4
 
5
+ export default function GuidelinePage({
6
+ annotatorName,
7
+ onChangeAnnotatorName,
8
+ onContinue,
9
+ apiBaseUrl,
10
+ onLookupResult
11
+ }) {
12
  const nameIsValid = annotatorName.trim().length > 0;
13
 
14
  // Lightweight name-collision guard: there's no login, so two different
 
33
  fetch(`${apiBaseUrl}/assignment/lookup?annotator=${encodeURIComponent(name)}`)
34
  .then((res) => (res.ok ? res.json() : { exists: false }))
35
  .then((data) => {
36
+ if (cancelled) return;
37
+ setCollisionInfo(data.exists ? data : null);
38
+ // A round only ever exists for a name that already passed
39
+ // training (see App.jsx's handleGuidelineLookupResult) - reuse
40
+ // this same lookup instead of firing a second request just to
41
+ // check that.
42
+ onLookupResult?.(data);
43
  })
44
  .catch(() => {
45
  // Fail open - this check is advisory, never block name entry on a lookup error.
 
51
  cancelled = true;
52
  clearTimeout(timer);
53
  };
54
+ }, [annotatorName, apiBaseUrl, onLookupResult]);
55
 
56
  const hasUnacknowledgedCollision = Boolean(collisionInfo) && !collisionAcknowledged;
57
 
 
67
  placeholder="Your name"
68
  />
69
  </label>
70
+ <p className="muted validation-hint">
71
+ Use the same name every time you return — an in-progress round resumes
72
+ automatically, saved answers included.
73
+ </p>
74
  {!nameIsValid && <p className="muted validation-hint">Enter your name to continue.</p>}
75
 
76
  {collisionInfo && (
77
  <div className="callout collision-warning">
78
+ <span className="k">Is this you?</span>
79
  An assignment already exists for &ldquo;{annotatorName.trim()}&rdquo; (
80
+ {collisionInfo.completed} of {collisionInfo.total} completed) — if so, you'll
81
+ resume with your saved answers already filled in.
82
  <label className="confirmation-row">
83
  <input
84
  type="checkbox"