diff weatherlog/logger_test.py @ 14:c01f9929ae38

Make logger and HTTP writer more general and resilient. This makes the logger and HTTP writer more general, by removing any dependency upon the exact data type they are writing. They can now handle any type of BSON-serializable dict, and track what they have sent by keeping track of the last *byte*, not the last timestamp.
author Paul Fisher <paul@pfish.zone>
date Tue, 15 Oct 2019 22:40:24 -0400
parents 8a350ec1aa78
children 770215590d80
line wrap: on
line diff
--- a/weatherlog/logger_test.py	Sun Sep 29 12:11:16 2019 -0400
+++ b/weatherlog/logger_test.py	Tue Oct 15 22:40:24 2019 -0400
@@ -46,6 +46,9 @@
     return datetime.datetime.utcfromtimestamp(t).replace(tzinfo=pytz.UTC)
 
 
+bs = logger.bson_encode
+
+
 class LoggerTest(unittest.TestCase):
 
     maxDiff = None
@@ -65,24 +68,24 @@
             logger.BufferedLogger(self.temp_dir.name, writer)
         ) as bl:
             bl.WAIT_TIME = 0.2
-            bl.write(types.Reading(ts(3), 1, 2))
-            bl.write(types.Reading(ts(6), 4, 5))
-            bl.write(types.Reading(ts(8), 10, 9))
+            bl.write({'first': 'item'})
+            bl.write({'entry': 2})
+            bl.write({'thing': 'three'})
             bl.start()
             time.sleep(1)  # Ensure that we get an entire logger cycle in.
 
         self.assertEqual(
             writer.writes,
             [
-                (types.Reading(ts(3), 1, 2), types.Reading(ts(6), 4, 5)),
-                (types.Reading(ts(8), 10, 9),),
+                (bs({'first': 'item'}), bs({'entry': 2})),
+                (bs({'thing': 'three'}),),
             ],
         )
-        self.assertEqual(self._read_last_sent(), '8.0')
+        self.assertEqual(self._read_last_sent(), '59')
         self.assertEqual(self._read_bsons(), [
-            dict(sample_time=ts(3), temp_c=1, rh_pct=2),
-            dict(sample_time=ts(6), temp_c=4, rh_pct=5),
-            dict(sample_time=ts(8), temp_c=10, rh_pct=9),
+            {'first': 'item'},
+            {'entry': 2},
+            {'thing': 'three'},
         ])
 
     def test_append_and_resume(self):
@@ -92,10 +95,10 @@
         ]
         with (self.temp_path / logger.BSON_FILENAME).open('wb') as outfile:
             for value in existing_values:
-                outfile.write(logger.bson_encode(value))
+                outfile.write(bs(value))
             outfile.write(b'non-BSON garbage')
 
-        with (self.temp_path / logger.LAST_SENT_FILENAME).open('w') as outfile:
+        with (self.temp_path / logger.OLD_LAST_TS).open('w') as outfile:
             outfile.write('10')
 
         writer = FakeWriter()
@@ -105,18 +108,50 @@
             bl.WAIT_TIME = 0.2
             bl.start()
             time.sleep(0.5)
-            bl.write(types.Reading(ts(99), temp_c=-40, rh_pct=2))
+            bl.write({'some new': 'entry'})
             time.sleep(0.5)
 
-        self.assertEqual(self._read_last_sent(), '99.0')
+        self.assertEqual(self._read_last_sent(), '125')
         self.assertEqual(self._read_bsons(), [
             dict(sample_time=ts(10), temp_c=20, rh_pct=30),
             dict(sample_time=ts(60), temp_c=10, rh_pct=99),
-            dict(sample_time=ts(99), temp_c=-40, rh_pct=2),
+            {'some new': 'entry'},
         ])
         self.assertEqual(list(itertools.chain.from_iterable(writer.writes)), [
-            types.Reading(ts(60), 10, 99),
-            types.Reading(ts(99), -40, 2),
+            bs(dict(sample_time=ts(60), temp_c=10, rh_pct=99)),
+            bs({'some new': 'entry'}),
+        ])
+
+    def test_resume_from_byte(self):
+        existing_values = [
+            {'old': 'value'},
+            {'unsent': 'value'},
+        ]
+        with (self.temp_path / logger.BSON_FILENAME).open('wb') as outfile:
+            for value in existing_values:
+                outfile.write(bs(value))
+            outfile.write(b'non-BSON garbage')
+        with (self.temp_path / logger.START_BYTE).open('w') as outfile:
+            outfile.write('20')  # immediately after 'old: value'
+
+        writer = FakeWriter()
+        with contextlib.closing(
+            logger.BufferedLogger(str(self.temp_path), writer)
+        ) as bl:
+            bl.WAIT_TIME = 0.2
+            bl.start()
+            bl.write({'some new': 'entry'})
+            time.sleep(0.5)
+
+        self.assertEqual(self._read_last_sent(), '68')
+        self.assertEqual(self._read_bsons(), [
+            {'old': 'value'},
+            {'unsent': 'value'},
+            {'some new': 'entry'},
+        ])
+        self.assertEqual(list(itertools.chain.from_iterable(writer.writes)), [
+            bs({'unsent': 'value'}),
+            bs({'some new': 'entry'}),
         ])
 
     def test_send_failure(self):
@@ -126,34 +161,34 @@
         ) as bl:
             bl.WAIT_TIME = 0.2
             bl.start()
-            bl.write(types.Reading(ts(1337), 420, 69))
+            bl.write({'cool write': 'succeeds'})
             time.sleep(0.5)
             writer.is_bad = True
-            bl.write(types.Reading(ts(31337), 666, 999))
+            bl.write({'bad write': 'fails'})
             time.sleep(0.5)
 
-            self.assertEqual(self._read_last_sent(), '1337.0')
+            self.assertEqual(self._read_last_sent(), '30')
             self.assertEqual(
-                writer.writes, [(types.Reading(ts(1337), 420, 69),)])
+                writer.writes, [(bs({'cool write': 'succeeds'}),)])
             self.assertEqual(self._read_bsons(), [
-                dict(sample_time=ts(1337), temp_c=420, rh_pct=69),
-                dict(sample_time=ts(31337), temp_c=666, rh_pct=999),
+                {'cool write': 'succeeds'},
+                {'bad write': 'fails'},
             ])
 
             # Ensure that we resume writing again when the condition clears.
 
             writer.is_bad = False
             time.sleep(0.5)
-        self.assertEqual(self._read_last_sent(), '31337.0')
+        self.assertEqual(self._read_last_sent(), '56')
         self.assertEqual(
             writer.writes,
             [
-                (types.Reading(ts(1337), 420, 69),),
-                (types.Reading(ts(31337), 666, 999),),
+                (bs({'cool write': 'succeeds'}),),
+                (bs({'bad write': 'fails'}),),
             ])
         self.assertEqual(self._read_bsons(), [
-            dict(sample_time=ts(1337), temp_c=420, rh_pct=69),
-            dict(sample_time=ts(31337), temp_c=666, rh_pct=999),
+            {'cool write': 'succeeds'},
+            {'bad write': 'fails'},
         ])
 
     def test_fail_upon_lock(self):
@@ -177,8 +212,48 @@
         # Test that it works after the lock is released.
         logger.BufferedLogger(str(self.temp_path), FakeWriter()).close()
 
+    def test_upgrade_last_sent(self):
+        for (timestamp, byte_count) in [
+            ('5', '0'),
+            ('20', '52'),
+            ('30', '78'),
+        ]:
+            bson_file = self.temp_path / logger.BSON_FILENAME
+            with bson_file.open('wb') as outfile:
+                outfile.write(logger.bson_encode(dict(sample_time=ts(10))))
+                outfile.write(logger.bson_encode(dict(sample_time=ts(20))))
+                outfile.write(logger.bson_encode(dict(sample_time=ts(30))))
+                outfile.write(b'some bogus data')
+            last_sent = self.temp_path / logger.OLD_LAST_TS
+            with last_sent.open('w') as outfile:
+                outfile.write(timestamp)
+            start_byte = self.temp_path / logger.START_BYTE
+            with bson_file.open('r+b') as infile:
+                logger._read_unsent_and_upgrade(
+                    infile, last_sent, start_byte)
+            self.assertFalse(last_sent.exists())
+            with start_byte.open('r') as infile:
+                self.assertEqual(infile.read(), byte_count)
+
+    def test_upgrade_last_sent_no_last_sent(self):
+        bson_file = self.temp_path / logger.BSON_FILENAME
+        with bson_file.open('wb') as outfile:
+            outfile.write(logger.bson_encode(dict(sample_time=ts(10))))
+            outfile.write(logger.bson_encode(dict(sample_time=ts(20))))
+            outfile.write(logger.bson_encode(dict(sample_time=ts(30))))
+        last_sent = self.temp_path / logger.OLD_LAST_TS
+        start_byte = self.temp_path / logger.START_BYTE
+        with start_byte.open('w') as outfile:
+            outfile.write('untouched')
+        with bson_file.open('r+b') as infile:
+            logger._read_unsent_and_upgrade(
+                infile, last_sent, start_byte)
+        self.assertFalse(last_sent.exists())
+        with start_byte.open('r') as infile:
+            self.assertEqual(infile.read(), 'untouched')
+
     def _read_last_sent(self):
-        with (self.temp_path / logger.LAST_SENT_FILENAME).open('r') as infile:
+        with (self.temp_path / logger.START_BYTE).open('r') as infile:
             return infile.read()
 
     def _read_bsons(self):