Mercurial > personal > weatherlog
diff weatherlog/logger_test.py @ 14:c01f9929ae38
Make logger and HTTP writer more general and resilient.
This makes the logger and HTTP writer more general, by removing
any dependency upon the exact data type they are writing.
They can now handle any type of BSON-serializable dict,
and track what they have sent by keeping track of the last *byte*,
not the last timestamp.
author | Paul Fisher <paul@pfish.zone> |
---|---|
date | Tue, 15 Oct 2019 22:40:24 -0400 |
parents | 8a350ec1aa78 |
children | 770215590d80 |
line wrap: on
line diff
--- a/weatherlog/logger_test.py Sun Sep 29 12:11:16 2019 -0400 +++ b/weatherlog/logger_test.py Tue Oct 15 22:40:24 2019 -0400 @@ -46,6 +46,9 @@ return datetime.datetime.utcfromtimestamp(t).replace(tzinfo=pytz.UTC) +bs = logger.bson_encode + + class LoggerTest(unittest.TestCase): maxDiff = None @@ -65,24 +68,24 @@ logger.BufferedLogger(self.temp_dir.name, writer) ) as bl: bl.WAIT_TIME = 0.2 - bl.write(types.Reading(ts(3), 1, 2)) - bl.write(types.Reading(ts(6), 4, 5)) - bl.write(types.Reading(ts(8), 10, 9)) + bl.write({'first': 'item'}) + bl.write({'entry': 2}) + bl.write({'thing': 'three'}) bl.start() time.sleep(1) # Ensure that we get an entire logger cycle in. self.assertEqual( writer.writes, [ - (types.Reading(ts(3), 1, 2), types.Reading(ts(6), 4, 5)), - (types.Reading(ts(8), 10, 9),), + (bs({'first': 'item'}), bs({'entry': 2})), + (bs({'thing': 'three'}),), ], ) - self.assertEqual(self._read_last_sent(), '8.0') + self.assertEqual(self._read_last_sent(), '59') self.assertEqual(self._read_bsons(), [ - dict(sample_time=ts(3), temp_c=1, rh_pct=2), - dict(sample_time=ts(6), temp_c=4, rh_pct=5), - dict(sample_time=ts(8), temp_c=10, rh_pct=9), + {'first': 'item'}, + {'entry': 2}, + {'thing': 'three'}, ]) def test_append_and_resume(self): @@ -92,10 +95,10 @@ ] with (self.temp_path / logger.BSON_FILENAME).open('wb') as outfile: for value in existing_values: - outfile.write(logger.bson_encode(value)) + outfile.write(bs(value)) outfile.write(b'non-BSON garbage') - with (self.temp_path / logger.LAST_SENT_FILENAME).open('w') as outfile: + with (self.temp_path / logger.OLD_LAST_TS).open('w') as outfile: outfile.write('10') writer = FakeWriter() @@ -105,18 +108,50 @@ bl.WAIT_TIME = 0.2 bl.start() time.sleep(0.5) - bl.write(types.Reading(ts(99), temp_c=-40, rh_pct=2)) + bl.write({'some new': 'entry'}) time.sleep(0.5) - self.assertEqual(self._read_last_sent(), '99.0') + self.assertEqual(self._read_last_sent(), '125') self.assertEqual(self._read_bsons(), [ dict(sample_time=ts(10), temp_c=20, rh_pct=30), dict(sample_time=ts(60), temp_c=10, rh_pct=99), - dict(sample_time=ts(99), temp_c=-40, rh_pct=2), + {'some new': 'entry'}, ]) self.assertEqual(list(itertools.chain.from_iterable(writer.writes)), [ - types.Reading(ts(60), 10, 99), - types.Reading(ts(99), -40, 2), + bs(dict(sample_time=ts(60), temp_c=10, rh_pct=99)), + bs({'some new': 'entry'}), + ]) + + def test_resume_from_byte(self): + existing_values = [ + {'old': 'value'}, + {'unsent': 'value'}, + ] + with (self.temp_path / logger.BSON_FILENAME).open('wb') as outfile: + for value in existing_values: + outfile.write(bs(value)) + outfile.write(b'non-BSON garbage') + with (self.temp_path / logger.START_BYTE).open('w') as outfile: + outfile.write('20') # immediately after 'old: value' + + writer = FakeWriter() + with contextlib.closing( + logger.BufferedLogger(str(self.temp_path), writer) + ) as bl: + bl.WAIT_TIME = 0.2 + bl.start() + bl.write({'some new': 'entry'}) + time.sleep(0.5) + + self.assertEqual(self._read_last_sent(), '68') + self.assertEqual(self._read_bsons(), [ + {'old': 'value'}, + {'unsent': 'value'}, + {'some new': 'entry'}, + ]) + self.assertEqual(list(itertools.chain.from_iterable(writer.writes)), [ + bs({'unsent': 'value'}), + bs({'some new': 'entry'}), ]) def test_send_failure(self): @@ -126,34 +161,34 @@ ) as bl: bl.WAIT_TIME = 0.2 bl.start() - bl.write(types.Reading(ts(1337), 420, 69)) + bl.write({'cool write': 'succeeds'}) time.sleep(0.5) writer.is_bad = True - bl.write(types.Reading(ts(31337), 666, 999)) + bl.write({'bad write': 'fails'}) time.sleep(0.5) - self.assertEqual(self._read_last_sent(), '1337.0') + self.assertEqual(self._read_last_sent(), '30') self.assertEqual( - writer.writes, [(types.Reading(ts(1337), 420, 69),)]) + writer.writes, [(bs({'cool write': 'succeeds'}),)]) self.assertEqual(self._read_bsons(), [ - dict(sample_time=ts(1337), temp_c=420, rh_pct=69), - dict(sample_time=ts(31337), temp_c=666, rh_pct=999), + {'cool write': 'succeeds'}, + {'bad write': 'fails'}, ]) # Ensure that we resume writing again when the condition clears. writer.is_bad = False time.sleep(0.5) - self.assertEqual(self._read_last_sent(), '31337.0') + self.assertEqual(self._read_last_sent(), '56') self.assertEqual( writer.writes, [ - (types.Reading(ts(1337), 420, 69),), - (types.Reading(ts(31337), 666, 999),), + (bs({'cool write': 'succeeds'}),), + (bs({'bad write': 'fails'}),), ]) self.assertEqual(self._read_bsons(), [ - dict(sample_time=ts(1337), temp_c=420, rh_pct=69), - dict(sample_time=ts(31337), temp_c=666, rh_pct=999), + {'cool write': 'succeeds'}, + {'bad write': 'fails'}, ]) def test_fail_upon_lock(self): @@ -177,8 +212,48 @@ # Test that it works after the lock is released. logger.BufferedLogger(str(self.temp_path), FakeWriter()).close() + def test_upgrade_last_sent(self): + for (timestamp, byte_count) in [ + ('5', '0'), + ('20', '52'), + ('30', '78'), + ]: + bson_file = self.temp_path / logger.BSON_FILENAME + with bson_file.open('wb') as outfile: + outfile.write(logger.bson_encode(dict(sample_time=ts(10)))) + outfile.write(logger.bson_encode(dict(sample_time=ts(20)))) + outfile.write(logger.bson_encode(dict(sample_time=ts(30)))) + outfile.write(b'some bogus data') + last_sent = self.temp_path / logger.OLD_LAST_TS + with last_sent.open('w') as outfile: + outfile.write(timestamp) + start_byte = self.temp_path / logger.START_BYTE + with bson_file.open('r+b') as infile: + logger._read_unsent_and_upgrade( + infile, last_sent, start_byte) + self.assertFalse(last_sent.exists()) + with start_byte.open('r') as infile: + self.assertEqual(infile.read(), byte_count) + + def test_upgrade_last_sent_no_last_sent(self): + bson_file = self.temp_path / logger.BSON_FILENAME + with bson_file.open('wb') as outfile: + outfile.write(logger.bson_encode(dict(sample_time=ts(10)))) + outfile.write(logger.bson_encode(dict(sample_time=ts(20)))) + outfile.write(logger.bson_encode(dict(sample_time=ts(30)))) + last_sent = self.temp_path / logger.OLD_LAST_TS + start_byte = self.temp_path / logger.START_BYTE + with start_byte.open('w') as outfile: + outfile.write('untouched') + with bson_file.open('r+b') as infile: + logger._read_unsent_and_upgrade( + infile, last_sent, start_byte) + self.assertFalse(last_sent.exists()) + with start_byte.open('r') as infile: + self.assertEqual(infile.read(), 'untouched') + def _read_last_sent(self): - with (self.temp_path / logger.LAST_SENT_FILENAME).open('r') as infile: + with (self.temp_path / logger.START_BYTE).open('r') as infile: return infile.read() def _read_bsons(self):