Mercurial > personal > weather-server

annotate weather_server/logfile_test.py @ 39:b77c8e7d2742

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Use zoneinfo rather than pytz.
author Paul Fisher <paul@pfish.zone>
date Tue, 01 Apr 2025 15:54:21 -0400
parents 9bc3687e1e5e
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
1 import contextlib
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
2 import datetime
24
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
3 import os.path
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
4 import pathlib
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
5 import tempfile
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
6 import unittest
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
7
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
8 import bson
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
9
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
10 from . import common
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
11 from . import logfile
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
12
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
13
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
14 def ts(n):
39
b77c8e7d2742 Use zoneinfo rather than pytz.
Paul Fisher <paul@pfish.zone>
parents: 31
diff changeset
15 return datetime.datetime.fromtimestamp(n, datetime.UTC)
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
16
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
17
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
18 class LoggerTest(unittest.TestCase):
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
19
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
20 maxDiff = None
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
21
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
22 def setUp(self):
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
23 super().setUp()
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
24 self.temp_dir = tempfile.TemporaryDirectory()
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
25 self.log_path = pathlib.Path(self.temp_dir.name) / 'test.bson'
31
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
26 self.index_path = pathlib.Path(str(self.log_path) + '.index.bson')
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
27
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
28 def tearDown(self):
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
29 self.temp_dir.cleanup()
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
30 super().tearDown()
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
31
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
32 def test_empty(self):
21
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
33 lg = logfile.Logger(
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
34 str(self.log_path), sample_field='x')
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
35 with contextlib.closing(lg) as logger:
31
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
36 self.assertEqual(logger.cached_data, ())
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
37
24
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
38 def test_fails_to_open(self):
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
39 with self.assertRaises(OSError):
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
40 logfile.Logger(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
41 os.path.join(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
42 self.temp_dir.name,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
43 'nonexistent-directory',
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
44 'bogus-filename'),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
45 sample_field='unimportant')
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
46
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
47 def test_del(self):
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
48 lg = logfile.Logger(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
49 str(self.log_path), sample_field='x')
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
50 del lg
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
51
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
52 def test_loading(self):
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
53 with self.log_path.open('wb') as outfile:
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
54 outfile.write(common.bson_encode(dict(
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
55 sample_time=ts(123),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
56 temp_c=420,
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
57 rh_pct=69,
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
58 ingest_time=ts(125),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
59 )))
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
60 outfile.write(b'garbage to ignore')
21
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
61 with contextlib.closing(logfile.Logger(
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
62 str(self.log_path),
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
63 sample_field='sample_time',
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
64 )) as logger:
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
65 self.assertEqual(
31
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
66 logger.cached_data,
21
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
67 (
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
68 dict(
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
69 sample_time=ts(123),
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
70 temp_c=420,
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
71 rh_pct=69,
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
72 ingest_time=ts(125)),
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
73 ))
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
74
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
75 def test_writing(self):
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
76 with self.log_path.open('wb') as outfile:
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
77 outfile.write(common.bson_encode(dict(
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
78 sample_time=ts(123),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
79 temp_c=420,
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
80 rh_pct=69,
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
81 ingest_time=ts(125),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
82 )))
21
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
83 with contextlib.closing(logfile.Logger(
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
84 str(self.log_path),
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
85 sample_field='sample_time',
31
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
86 cached_entries=4,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
87 index_gap=2,
21
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
88 )) as logger:
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
89 logger.write_rows([
21
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
90 # Ignored, since it's older than the newest entry.
24
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
91 dict(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
92 sample_time=ts(100),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
93 temp_c=999,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
94 rh_pct=666,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
95 ingest_time=ts(101),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
96 ),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
97 dict(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
98 sample_time=ts(125),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
99 temp_c=333,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
100 rh_pct=777,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
101 ingest_time=ts(200),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
102 ),
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
103 ])
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
104 self.assertEqual(
31
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
105 logger.cached_data,
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
106 (
24
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
107 dict(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
108 sample_time=ts(123),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
109 temp_c=420,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
110 rh_pct=69,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
111 ingest_time=ts(125),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
112 ),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
113 dict(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
114 sample_time=ts(125),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
115 temp_c=333,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
116 rh_pct=777,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
117 ingest_time=ts(200),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
118 ),
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
119 )
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
120 )
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
121
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
122 self.assertEqual(self.read_bsons(), [
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
123 dict(
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
124 sample_time=ts(123),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
125 temp_c=420,
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
126 rh_pct=69,
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
127 ingest_time=ts(125),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
128 ),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
129 dict(
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
130 sample_time=ts(125),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
131 temp_c=333,
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
132 rh_pct=777,
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
133 ingest_time=ts(200),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
134 ),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
135 ])
31
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
136 self.assertEqual(self.read_index(), [
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
137 dict(entry_count=0, entry_key=ts(123), byte=0),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
138 ])
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
139
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
140 def test_outside_writes(self):
21
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
141 with contextlib.closing(logfile.Logger(
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
142 str(self.log_path),
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
143 sample_field='sample_time',
31
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
144 index_gap=2,
21
beb42c835c52 Make weather server handle arbitrary data:
Paul Fisher <paul@pfish.zone>
parents: 0
diff changeset
145 )) as logger:
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
146 logger.write_rows([
24
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
147 dict(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
148 sample_time=ts(100),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
149 temp_c=999,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
150 rh_pct=666,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
151 ingest_time=ts(101),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
152 ),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
153 dict(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
154 sample_time=ts(125),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
155 temp_c=333,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
156 rh_pct=777,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
157 ingest_time=ts(200),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
158 ),
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
159 ])
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
160 with self.log_path.open('ab') as outfile:
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
161 outfile.write(common.bson_encode(dict(
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
162 sample_time=ts(1024),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
163 temp_c=256,
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
164 rh_pct=128,
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
165 ingest_time=ts(4096),
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
166 )))
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
167 outfile.flush()
31
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
168 # Empty write to update the index.
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
169 logger.write_rows([])
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
170 self.assertEqual(logger.cached_data, (
24
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
171 dict(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
172 sample_time=ts(100),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
173 temp_c=999,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
174 rh_pct=666,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
175 ingest_time=ts(101),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
176 ),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
177 dict(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
178 sample_time=ts(125),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
179 temp_c=333,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
180 rh_pct=777,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
181 ingest_time=ts(200),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
182 ),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
183 dict(
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
184 sample_time=ts(1024),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
185 temp_c=256,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
186 rh_pct=128,
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
187 ingest_time=ts(4096),
20c8ec56e447 logfile: Pull logfile thread out of Logger.
Paul Fisher <paul@pfish.zone>
parents: 21
diff changeset
188 ),
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
189 ))
31
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
190 self.assertEqual(self.read_bsons(), [
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
191 dict(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
192 sample_time=ts(100),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
193 temp_c=999,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
194 rh_pct=666,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
195 ingest_time=ts(101),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
196 ),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
197 dict(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
198 sample_time=ts(125),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
199 temp_c=333,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
200 rh_pct=777,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
201 ingest_time=ts(200),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
202 ),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
203 dict(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
204 sample_time=ts(1024),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
205 temp_c=256,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
206 rh_pct=128,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
207 ingest_time=ts(4096),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
208 ),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
209 ])
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
210 self.assertEqual(self.read_index(), [
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
211 dict(entry_count=0, entry_key=ts(100), byte=0),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
212 dict(entry_count=2, entry_key=ts(1024), byte=142),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
213 ])
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
214
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
215 def test_lots_of_outside_writes(self):
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
216 with contextlib.closing(logfile.Logger(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
217 str(self.log_path),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
218 sample_field='key',
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
219 cached_entries=4,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
220 index_gap=2,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
221 )) as logger:
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
222 logger.write_rows(make_messages(10))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
223 with contextlib.closing(logfile.Logger(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
224 str(self.log_path),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
225 sample_field='key',
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
226 cached_entries=4,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
227 index_gap=2,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
228 )) as other_logger:
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
229 other_logger.write_rows(make_messages(10, 10))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
230 logger.write_rows(make_messages(1, 20))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
231 self.assertEqual(logger.cached_data, (
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
232 dict(key='0011', value=17),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
233 dict(key='0012', value=18),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
234 dict(key='0013', value=19),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
235 dict(key='0014', value=20),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
236 ))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
237 self.assertEqual(self.read_bsons(), make_messages(21))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
238 self.assertEqual(self.read_index(), [
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
239 dict(entry_count=0, entry_key='0000', byte=0),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
240 dict(entry_count=2, entry_key='0002', byte=60),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
241 dict(entry_count=4, entry_key='0004', byte=120),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
242 dict(entry_count=6, entry_key='0006', byte=180),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
243 dict(entry_count=8, entry_key='0008', byte=240),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
244 dict(entry_count=10, entry_key='000a', byte=300),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
245 dict(entry_count=12, entry_key='000c', byte=360),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
246 dict(entry_count=14, entry_key='000e', byte=420),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
247 dict(entry_count=16, entry_key='0010', byte=480),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
248 dict(entry_count=18, entry_key='0012', byte=540),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
249 dict(entry_count=20, entry_key='0014', byte=600),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
250 ])
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
251
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
252 def test_outside_writes_that_dont_update_index(self):
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
253 with contextlib.closing(logfile.Logger(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
254 str(self.log_path),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
255 sample_field='key',
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
256 cached_entries=4,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
257 index_gap=2,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
258 )) as logger:
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
259 logger.write_rows(make_messages(10))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
260 with self.log_path.open('ab') as outfile:
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
261 for item in make_messages(16, 256):
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
262 outfile.write(common.bson_encode(item))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
263 self.assertEqual(logger.cached_data, (
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
264 dict(key='010c', value=268),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
265 dict(key='010d', value=269),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
266 dict(key='010e', value=270),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
267 dict(key='010f', value=271),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
268 ))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
269 self.assertEqual(self.read_bsons(), [
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
270 dict(key='0000', value=0),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
271 dict(key='0001', value=1),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
272 dict(key='0002', value=2),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
273 dict(key='0003', value=3),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
274 dict(key='0004', value=4),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
275 dict(key='0005', value=5),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
276 dict(key='0006', value=6),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
277 dict(key='0007', value=7),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
278 dict(key='0008', value=8),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
279 dict(key='0009', value=9),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
280 dict(key='0100', value=256),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
281 dict(key='0101', value=257),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
282 dict(key='0102', value=258),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
283 dict(key='0103', value=259),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
284 dict(key='0104', value=260),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
285 dict(key='0105', value=261),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
286 dict(key='0106', value=262),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
287 dict(key='0107', value=263),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
288 dict(key='0108', value=264),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
289 dict(key='0109', value=265),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
290 dict(key='010a', value=266),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
291 dict(key='010b', value=267),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
292 dict(key='010c', value=268),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
293 dict(key='010d', value=269),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
294 dict(key='010e', value=270),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
295 dict(key='010f', value=271),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
296 ])
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
297 self.assertEqual(self.read_index(), [
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
298 dict(entry_count=0, entry_key='0000', byte=0),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
299 dict(entry_count=2, entry_key='0002', byte=60),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
300 dict(entry_count=4, entry_key='0004', byte=120),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
301 dict(entry_count=6, entry_key='0006', byte=180),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
302 dict(entry_count=8, entry_key='0008', byte=240),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
303 ])
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
304
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
305 def test_outside_write_but_dont_skip(self):
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
306 with contextlib.closing(logfile.Logger(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
307 str(self.log_path),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
308 sample_field='key',
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
309 cached_entries=10,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
310 index_gap=4,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
311 )) as logger:
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
312 logger.write_rows(make_messages(10))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
313 with contextlib.closing(logfile.Logger(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
314 str(self.log_path),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
315 sample_field='key',
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
316 cached_entries=10,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
317 index_gap=4,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
318 )) as other_logger:
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
319 other_logger.write_rows(make_messages(5, 10))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
320 self.assertEqual(logger.cached_data, (
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
321 dict(key='0005', value=5),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
322 dict(key='0006', value=6),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
323 dict(key='0007', value=7),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
324 dict(key='0008', value=8),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
325 dict(key='0009', value=9),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
326 dict(key='000a', value=10),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
327 dict(key='000b', value=11),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
328 dict(key='000c', value=12),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
329 dict(key='000d', value=13),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
330 dict(key='000e', value=14),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
331 ))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
332 logger.write_rows(make_messages(5, 15))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
333 self.assertEqual(self.read_index(), [
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
334 dict(entry_count=0, entry_key='0000', byte=0),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
335 dict(entry_count=4, entry_key='0004', byte=120),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
336 dict(entry_count=8, entry_key='0008', byte=240),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
337 dict(entry_count=12, entry_key='000c', byte=360),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
338 dict(entry_count=16, entry_key='0010', byte=480),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
339 ])
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
340 logger.write_rows(make_messages(1, 20))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
341 self.assertEqual(self.read_index(), [
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
342 dict(entry_count=0, entry_key='0000', byte=0),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
343 dict(entry_count=4, entry_key='0004', byte=120),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
344 dict(entry_count=8, entry_key='0008', byte=240),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
345 dict(entry_count=12, entry_key='000c', byte=360),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
346 dict(entry_count=16, entry_key='0010', byte=480),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
347 dict(entry_count=20, entry_key='0014', byte=600),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
348 ])
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
349
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
350 def test_start_and_skip(self):
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
351 with contextlib.closing(logfile.Logger(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
352 str(self.log_path),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
353 sample_field='key',
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
354 cached_entries=4,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
355 index_gap=2,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
356 )) as logger:
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
357 logger.write_rows(make_messages(10))
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
358 with contextlib.closing(logfile.Logger(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
359 str(self.log_path),
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
360 sample_field='key',
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
361 cached_entries=4,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
362 index_gap=2,
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
363 )) as other_logger:
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
364 self.assertEqual(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
365 other_logger.cached_data, tuple(make_messages(4, 6)))
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
366
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
367 def read_bsons(self):
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
368 with self.log_path.open('rb') as infile:
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
369 return bson.decode_all(
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
370 infile.read(), common.BSON_OPTIONS)
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
371
31
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
372 def read_index(self):
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
373 with self.index_path.open('rb') as infile:
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
374 return bson.decode_all(
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
375 infile.read(), common.BSON_OPTIONS)
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
376
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
377
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
378 def make_messages(count, start=0):
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
379 return [
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
380 dict(key=format(n, '04x'), value=n)
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
381 for n in range(start, start + count)]
9bc3687e1e5e logfile: Add an index, and don't keep everything in RAM.
Paul Fisher <paul@pfish.zone>
parents: 24
diff changeset
382
0
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
383
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
384 if __name__ == '__main__':
efe7a1eff167 Create initial logger for weather server.
Paul Fisher <paul@pfish.zone>
parents:
diff changeset
385 unittest.main()