Skip to content
This repository was archived by the owner on Jun 30, 2022. It is now read-only.

Commit 8809b6f

Browse files
fahndrichaaltay
authored andcommitted
At 0 progress don't pretend to be 1 byte done.
----Release Notes---- At 0 progress don't pretend to be 1 byte done. Until first record is done, progress should remain 0, otherwise wild interpolation of work estimate ensues. Fraction consumed really should be named fraction_done or fraction_completed, because that is the progress signal required. [] ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=121706839
1 parent 4784ea5 commit 8809b6f

2 files changed

Lines changed: 9 additions & 8 deletions

File tree

google/cloud/dataflow/io/range_trackers.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ def try_split_at_position(self, split_offset):
101101
@property
102102
def fraction_consumed(self):
103103
with self.lock:
104-
fraction = ((1.0 * (self.last_record_start - self.start_position + 1) /
104+
fraction = ((1.0 * (self.last_record_start - self.start_position) /
105105
(self.stop_position - self.start_position)) if
106106
self.stop_position != self.start_position else 0.0)
107107

google/cloud/dataflow/io/range_trackers_test.py

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -110,25 +110,26 @@ def test_get_fraction_consumed_dense(self):
110110
tracker = range_trackers.OffsetRangeTracker(3, 6)
111111
self.assertEqual(0, tracker.fraction_consumed)
112112
self.assertTrue(tracker.try_return_record_at(True, 3))
113-
self.assertEqual(1.0 / 3, tracker.fraction_consumed)
113+
self.assertEqual(0.0, tracker.fraction_consumed)
114114
self.assertTrue(tracker.try_return_record_at(True, 4))
115-
self.assertEqual(2.0 / 3, tracker.fraction_consumed)
115+
self.assertEqual(1.0 / 3, tracker.fraction_consumed)
116116
self.assertTrue(tracker.try_return_record_at(True, 5))
117-
self.assertEqual(1.0, tracker.fraction_consumed)
117+
self.assertEqual(2.0 / 3, tracker.fraction_consumed)
118118
self.assertTrue(tracker.try_return_record_at(False, 6)) # non-split-point
119+
self.assertEqual(1.0, tracker.fraction_consumed)
119120
self.assertTrue(tracker.try_return_record_at(False, 7)) # non-split-point
120121
self.assertFalse(tracker.try_return_record_at(True, 7))
121122

122123
def test_get_fraction_consumed_sparse(self):
123124
tracker = range_trackers.OffsetRangeTracker(100, 200)
124125
self.assertEqual(0, tracker.fraction_consumed)
125126
self.assertTrue(tracker.try_return_record_at(True, 110))
126-
# Consumed positions through 110 = total 11 positions of 100.
127-
self.assertEqual(0.11, tracker.fraction_consumed)
127+
# Consumed positions through 110 = total 10 positions of 100 done.
128+
self.assertEqual(0.10, tracker.fraction_consumed)
128129
self.assertTrue(tracker.try_return_record_at(True, 150))
129-
self.assertEqual(0.51, tracker.fraction_consumed)
130+
self.assertEqual(0.50, tracker.fraction_consumed)
130131
self.assertTrue(tracker.try_return_record_at(True, 195))
131-
self.assertEqual(0.96, tracker.fraction_consumed)
132+
self.assertEqual(0.95, tracker.fraction_consumed)
132133

133134
def test_everything_with_unbounded_range(self):
134135
tracker = range_trackers.OffsetRangeTracker(

0 commit comments

Comments
 (0)