Mercurial > personal > weather-server

annotate weather_server/logfile_test.py @ 32:2faf3499a226

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Ignore lots of development miscellany.
author Paul Fisher <paul@pfish.zone>
date Sat, 12 Jun 2021 18:43:49 +0000
parents 9bc3687e1e5e
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
1 import contextlib
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
2 import datetime
24
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
3 import os.path
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
4 import pathlib
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
5 import tempfile
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
6 import unittest
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
7
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
8 import bson
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
9 import pytz
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
10
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
11 from . import common
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
12 from . import logfile
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
13
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
14
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
15 def ts(n):
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
16 return datetime.datetime.utcfromtimestamp(n).replace(tzinfo=pytz.UTC)
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
17
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
18
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
19 class LoggerTest(unittest.TestCase):
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
20
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
21 maxDiff = None
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
22
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
23 def setUp(self):
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
24 super().setUp()
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
25 self.temp_dir = tempfile.TemporaryDirectory()
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
26 self.log_path = pathlib.Path(self.temp_dir.name) / 'test.bson'
31
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
27 self.index_path = pathlib.Path(str(self.log_path) + '.index.bson')
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
28
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
29 def tearDown(self):
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
30 self.temp_dir.cleanup()
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
31 super().tearDown()
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
32
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
33 def test_empty(self):
21
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
34 lg = logfile.Logger(
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
35 str(self.log_path), sample_field='x')
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
36 with contextlib.closing(lg) as logger:
31
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
37 self.assertEqual(logger.cached_data, ())
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
38
24
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
39 def test_fails_to_open(self):
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
40 with self.assertRaises(OSError):
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
41 logfile.Logger(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
42 os.path.join(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
43 self.temp_dir.name,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
44 'nonexistent-directory',
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
45 'bogus-filename'),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
46 sample_field='unimportant')
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
47
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
48 def test_del(self):
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
49 lg = logfile.Logger(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
50 str(self.log_path), sample_field='x')
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
51 del lg
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
52
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
53 def test_loading(self):
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
54 with self.log_path.open('wb') as outfile:
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
55 outfile.write(common.bson_encode(dict(
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
56 sample_time=ts(123),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
57 temp_c=420,
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
58 rh_pct=69,
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
59 ingest_time=ts(125),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
60 )))
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
61 outfile.write(b'garbage to ignore')
21
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
62 with contextlib.closing(logfile.Logger(
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
63 str(self.log_path),
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
64 sample_field='sample_time',
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
65 )) as logger:
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
66 self.assertEqual(
31
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
67 logger.cached_data,
21
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
68 (
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
69 dict(
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
70 sample_time=ts(123),
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
71 temp_c=420,
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
72 rh_pct=69,
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
73 ingest_time=ts(125)),
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
74 ))
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
75
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
76 def test_writing(self):
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
77 with self.log_path.open('wb') as outfile:
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
78 outfile.write(common.bson_encode(dict(
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
79 sample_time=ts(123),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
80 temp_c=420,
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
81 rh_pct=69,
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
82 ingest_time=ts(125),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
83 )))
21
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
84 with contextlib.closing(logfile.Logger(
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
85 str(self.log_path),
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
86 sample_field='sample_time',
31
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
87 cached_entries=4,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
88 index_gap=2,
21
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
89 )) as logger:
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
90 logger.write_rows([
21
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
91 # Ignored, since it's older than the newest entry.
24
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
92 dict(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
93 sample_time=ts(100),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
94 temp_c=999,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
95 rh_pct=666,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
96 ingest_time=ts(101),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
97 ),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
98 dict(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
99 sample_time=ts(125),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
100 temp_c=333,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
101 rh_pct=777,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
102 ingest_time=ts(200),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
103 ),
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
104 ])
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
105 self.assertEqual(
31
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
106 logger.cached_data,
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
107 (
24
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
108 dict(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
109 sample_time=ts(123),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
110 temp_c=420,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
111 rh_pct=69,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
112 ingest_time=ts(125),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
113 ),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
114 dict(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
115 sample_time=ts(125),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
116 temp_c=333,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
117 rh_pct=777,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
118 ingest_time=ts(200),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
119 ),
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
120 )
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
121 )
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
122
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
123 self.assertEqual(self.read_bsons(), [
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
124 dict(
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
125 sample_time=ts(123),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
126 temp_c=420,
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
127 rh_pct=69,
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
128 ingest_time=ts(125),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
129 ),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
130 dict(
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
131 sample_time=ts(125),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
132 temp_c=333,
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
133 rh_pct=777,
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
134 ingest_time=ts(200),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
135 ),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
136 ])
31
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
137 self.assertEqual(self.read_index(), [
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
138 dict(entry_count=0, entry_key=ts(123), byte=0),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
139 ])
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
140
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
141 def test_outside_writes(self):
21
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
142 with contextlib.closing(logfile.Logger(
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
143 str(self.log_path),
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
144 sample_field='sample_time',
31
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
145 index_gap=2,
21
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
146 )) as logger:
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
147 logger.write_rows([
24
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
148 dict(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
149 sample_time=ts(100),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
150 temp_c=999,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
151 rh_pct=666,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
152 ingest_time=ts(101),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
153 ),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
154 dict(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
155 sample_time=ts(125),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
156 temp_c=333,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
157 rh_pct=777,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
158 ingest_time=ts(200),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
159 ),
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
160 ])
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
161 with self.log_path.open('ab') as outfile:
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
162 outfile.write(common.bson_encode(dict(
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
163 sample_time=ts(1024),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
164 temp_c=256,
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
165 rh_pct=128,
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
166 ingest_time=ts(4096),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
167 )))
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
168 outfile.flush()
31
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
169 # Empty write to update the index.
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
170 logger.write_rows([])
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
171 self.assertEqual(logger.cached_data, (
24
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
172 dict(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
173 sample_time=ts(100),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
174 temp_c=999,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
175 rh_pct=666,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
176 ingest_time=ts(101),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
177 ),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
178 dict(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
179 sample_time=ts(125),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
180 temp_c=333,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
181 rh_pct=777,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
182 ingest_time=ts(200),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
183 ),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
184 dict(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
185 sample_time=ts(1024),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
186 temp_c=256,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
187 rh_pct=128,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
188 ingest_time=ts(4096),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
189 ),
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
190 ))
31
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
191 self.assertEqual(self.read_bsons(), [
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
192 dict(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
193 sample_time=ts(100),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
194 temp_c=999,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
195 rh_pct=666,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
196 ingest_time=ts(101),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
197 ),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
198 dict(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
199 sample_time=ts(125),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
200 temp_c=333,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
201 rh_pct=777,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
202 ingest_time=ts(200),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
203 ),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
204 dict(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
205 sample_time=ts(1024),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
206 temp_c=256,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
207 rh_pct=128,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
208 ingest_time=ts(4096),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
209 ),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
210 ])
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
211 self.assertEqual(self.read_index(), [
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
212 dict(entry_count=0, entry_key=ts(100), byte=0),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
213 dict(entry_count=2, entry_key=ts(1024), byte=142),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
214 ])
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
215
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
216 def test_lots_of_outside_writes(self):
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
217 with contextlib.closing(logfile.Logger(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
218 str(self.log_path),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
219 sample_field='key',
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
220 cached_entries=4,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
221 index_gap=2,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
222 )) as logger:
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
223 logger.write_rows(make_messages(10))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
224 with contextlib.closing(logfile.Logger(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
225 str(self.log_path),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
226 sample_field='key',
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
227 cached_entries=4,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
228 index_gap=2,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
229 )) as other_logger:
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
230 other_logger.write_rows(make_messages(10, 10))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
231 logger.write_rows(make_messages(1, 20))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
232 self.assertEqual(logger.cached_data, (
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
233 dict(key='0011', value=17),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
234 dict(key='0012', value=18),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
235 dict(key='0013', value=19),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
236 dict(key='0014', value=20),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
237 ))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
238 self.assertEqual(self.read_bsons(), make_messages(21))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
239 self.assertEqual(self.read_index(), [
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
240 dict(entry_count=0, entry_key='0000', byte=0),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
241 dict(entry_count=2, entry_key='0002', byte=60),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
242 dict(entry_count=4, entry_key='0004', byte=120),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
243 dict(entry_count=6, entry_key='0006', byte=180),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
244 dict(entry_count=8, entry_key='0008', byte=240),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
245 dict(entry_count=10, entry_key='000a', byte=300),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
246 dict(entry_count=12, entry_key='000c', byte=360),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
247 dict(entry_count=14, entry_key='000e', byte=420),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
248 dict(entry_count=16, entry_key='0010', byte=480),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
249 dict(entry_count=18, entry_key='0012', byte=540),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
250 dict(entry_count=20, entry_key='0014', byte=600),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
251 ])
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
252
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
253 def test_outside_writes_that_dont_update_index(self):
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
254 with contextlib.closing(logfile.Logger(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
255 str(self.log_path),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
256 sample_field='key',
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
257 cached_entries=4,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
258 index_gap=2,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
259 )) as logger:
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
260 logger.write_rows(make_messages(10))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
261 with self.log_path.open('ab') as outfile:
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
262 for item in make_messages(16, 256):
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
263 outfile.write(common.bson_encode(item))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
264 self.assertEqual(logger.cached_data, (
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
265 dict(key='010c', value=268),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
266 dict(key='010d', value=269),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
267 dict(key='010e', value=270),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
268 dict(key='010f', value=271),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
269 ))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
270 self.assertEqual(self.read_bsons(), [
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
271 dict(key='0000', value=0),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
272 dict(key='0001', value=1),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
273 dict(key='0002', value=2),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
274 dict(key='0003', value=3),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
275 dict(key='0004', value=4),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
276 dict(key='0005', value=5),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
277 dict(key='0006', value=6),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
278 dict(key='0007', value=7),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
279 dict(key='0008', value=8),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
280 dict(key='0009', value=9),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
281 dict(key='0100', value=256),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
282 dict(key='0101', value=257),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
283 dict(key='0102', value=258),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
284 dict(key='0103', value=259),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
285 dict(key='0104', value=260),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
286 dict(key='0105', value=261),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
287 dict(key='0106', value=262),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
288 dict(key='0107', value=263),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
289 dict(key='0108', value=264),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
290 dict(key='0109', value=265),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
291 dict(key='010a', value=266),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
292 dict(key='010b', value=267),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
293 dict(key='010c', value=268),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
294 dict(key='010d', value=269),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
295 dict(key='010e', value=270),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
296 dict(key='010f', value=271),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
297 ])
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
298 self.assertEqual(self.read_index(), [
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
299 dict(entry_count=0, entry_key='0000', byte=0),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
300 dict(entry_count=2, entry_key='0002', byte=60),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
301 dict(entry_count=4, entry_key='0004', byte=120),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
302 dict(entry_count=6, entry_key='0006', byte=180),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
303 dict(entry_count=8, entry_key='0008', byte=240),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
304 ])
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
305
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
306 def test_outside_write_but_dont_skip(self):
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
307 with contextlib.closing(logfile.Logger(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
308 str(self.log_path),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
309 sample_field='key',
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
310 cached_entries=10,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
311 index_gap=4,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
312 )) as logger:
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
313 logger.write_rows(make_messages(10))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
314 with contextlib.closing(logfile.Logger(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
315 str(self.log_path),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
316 sample_field='key',
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
317 cached_entries=10,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
318 index_gap=4,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
319 )) as other_logger:
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
320 other_logger.write_rows(make_messages(5, 10))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
321 self.assertEqual(logger.cached_data, (
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
322 dict(key='0005', value=5),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
323 dict(key='0006', value=6),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
324 dict(key='0007', value=7),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
325 dict(key='0008', value=8),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
326 dict(key='0009', value=9),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
327 dict(key='000a', value=10),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
328 dict(key='000b', value=11),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
329 dict(key='000c', value=12),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
330 dict(key='000d', value=13),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
331 dict(key='000e', value=14),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
332 ))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
333 logger.write_rows(make_messages(5, 15))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
334 self.assertEqual(self.read_index(), [
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
335 dict(entry_count=0, entry_key='0000', byte=0),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
336 dict(entry_count=4, entry_key='0004', byte=120),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
337 dict(entry_count=8, entry_key='0008', byte=240),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
338 dict(entry_count=12, entry_key='000c', byte=360),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
339 dict(entry_count=16, entry_key='0010', byte=480),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
340 ])
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
341 logger.write_rows(make_messages(1, 20))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
342 self.assertEqual(self.read_index(), [
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
343 dict(entry_count=0, entry_key='0000', byte=0),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
344 dict(entry_count=4, entry_key='0004', byte=120),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
345 dict(entry_count=8, entry_key='0008', byte=240),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
346 dict(entry_count=12, entry_key='000c', byte=360),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
347 dict(entry_count=16, entry_key='0010', byte=480),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
348 dict(entry_count=20, entry_key='0014', byte=600),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
349 ])
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
350
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
351 def test_start_and_skip(self):
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
352 with contextlib.closing(logfile.Logger(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
353 str(self.log_path),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
354 sample_field='key',
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
355 cached_entries=4,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
356 index_gap=2,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
357 )) as logger:
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
358 logger.write_rows(make_messages(10))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
359 with contextlib.closing(logfile.Logger(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
360 str(self.log_path),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
361 sample_field='key',
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
362 cached_entries=4,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
363 index_gap=2,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
364 )) as other_logger:
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
365 self.assertEqual(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
366 other_logger.cached_data, tuple(make_messages(4, 6)))
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
367
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
368 def read_bsons(self):
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
369 with self.log_path.open('rb') as infile:
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
370 return bson.decode_all(
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
371 infile.read(), common.BSON_OPTIONS)
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
372
31
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
373 def read_index(self):
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
374 with self.index_path.open('rb') as infile:
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
375 return bson.decode_all(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
376 infile.read(), common.BSON_OPTIONS)
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
377
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
378
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
379 def make_messages(count, start=0):
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
380 return [
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
381 dict(key=format(n, '04x'), value=n)
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
382 for n in range(start, start + count)]
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
383
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
384
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
385 if __name__ == '__main__':
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
386 unittest.main()