Package osh :: Module priorityqueue
[frames] | no frames]

Source Code for Module osh.priorityqueue

  1  # osh 
  2  # Copyright (C) Jack Orenstein <jao@geophile.com> 
  3  # 
  4  # This program is free software; you can redistribute it and/or modify 
  5  # it under the terms of the GNU General Public License as published by 
  6  # the Free Software Foundation; either version 2 of the License, or 
  7  # (at your option) any later version. 
  8  # 
  9  # This program is distributed in the hope that it will be useful, 
 10  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 11  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
 12  # GNU General Public License for more details. 
 13  # 
 14  # You should have received a copy of the GNU General Public License 
 15  # along with this program; if not, write to the Free Software 
 16  # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 
 17   
 18  import threading 
 19   
 20  # import trace 
 21  # trace.on('/tmp/trace.txt') 
 22  # trace = trace.trace 
 23   
24 -class SpecialValue(object):
25 26 _name = None 27 _value = None 28
29 - def __init__(self, name, value):
30 self._name = name 31 self._value = value
32
33 - def __repr__(self):
34 return self._name
35
36 - def __lt__(self, other):
37 return self._value < other._value
38
39 - def __le__(self, other):
40 return self._value <= other._value
41
42 - def __gt__(self, other):
43 return self._value > other._value
44
45 - def __ge__(self, other):
46 return self._value >= other._value
47
48 - def __eq__(self, other):
49 return self._value == other._value
50
51 - def __ne__(self, other):
52 return self._value != other._value
53 54 MINUS_INFINITY = SpecialValue('MINUS_INFINITY', 0) 55 INFINITY = SpecialValue('INFINITY', 1) 56 DEBUG = True 57 WRITE_BUFFER_CAPACITY = 1000 58 LOCK_WAIT_TIME = 1.0 59
60 -def _describe_list(x):
61 n = len(x) 62 if n == 0: 63 boundaries = '' 64 elif n == 1: 65 boundaries = str(x[0]) 66 elif n == 2: 67 boundaries = '%s, %s' % (x[0], x[1]) 68 else: 69 boundaries = '%s .. %s' % (x[0], x[-1]) 70 return '%s[%s]' % (len(x), boundaries)
71
72 -class PriorityQueue(object):
73 74 _nodes = None 75 _inputs = None 76 _first_input = None 77 _key = None 78
79 - def __init__(self, key, inputs):
80 self._key = key 81 self._inputs = inputs 82 # Compute number of leaves and nodes 83 leaves = 1 84 while leaves < self._inputs: 85 leaves *= 2 86 nodes = 2 * leaves - 1 87 # Set up to create tree 88 self._first_input = leaves - 1 89 self._nodes = [] 90 # Create interior nodes 91 for i in xrange(self._first_input): 92 self._nodes.append(_InteriorNode(self)) 93 # Create input nodes 94 for i in xrange(self._inputs): 95 self._nodes.append(_InputNode(self, i)) 96 # Create filler nodes 97 while len(self._nodes) < nodes: 98 self._nodes.append(_FillerNode(self)) 99 # Starting with filler nodes, propagate INFINITY up the tree. In general, if self is 100 # INFINITY and self has a left sibling, and the left sibling is INFINITY, then 101 # propagate up. 102 for i in xrange(self._first_input + self._inputs, len(self._nodes)): 103 node = self._nodes[i] 104 while ((not node.is_root) and 105 node.parent.right is node and 106 type(node.parent.left) is not _InputNode and # can't check input node content 107 # before loading. 108 node.parent.left.content is INFINITY): 109 node = node.parent 110 node.content = INFINITY
111
112 - def __repr__(self):
113 return 'PriorityQueue<%s>' % id(self)
114
115 - def __iter__(self):
116 # Promote until root is not MINUS_INFINITY. 117 root = self._nodes[0] 118 while root.content is MINUS_INFINITY: 119 root.promote() 120 output = root.content 121 while output is not INFINITY: 122 # trace('%s: yielding %s' % (self, output)) 123 yield output 124 root.promote() 125 output = root.content
126
127 - def add(self, index, object):
128 # trace('%s: add to %s: %s' % (self, index, object)) 129 self._nodes[self._first_input + index].add(object)
130
131 - def done(self, index):
132 self.add(index, INFINITY)
133
134 - def dump(self, label = None):
135 if label: 136 label = [label] 137 else: 138 label = [] 139 return '\n'.join(label + ['%s: %s' % (node._id, str(node)) for node in self._nodes])
140 141
142 -class _Node(object):
143 144 _priority_queue = None 145 _key = None 146 _id = None 147
148 - def __init__(self, priority_queue):
149 self._priority_queue = priority_queue 150 self._key = priority_queue._key 151 self._id = len(priority_queue._nodes)
152
153 - def __repr__(self):
154 return '%s#%s(%s)' % (self.node_type(), self._id, self.content)
155
156 - def _compare(self, x, y):
157 if type(x) is type(y): 158 # Both SpecialValue or both not SpecialValue 159 if isinstance(x, SpecialValue): 160 c = cmp(x, y) 161 else: 162 key = self._key 163 kx = key(x) 164 ky = key(y) 165 c = cmp(kx, ky) 166 elif x is INFINITY: 167 c = 1 168 elif y is INFINITY: 169 c = -1 170 elif x is MINUS_INFINITY: 171 c = -1 172 elif y is MINUS_INFINITY: 173 c = 1 174 else: 175 assert False, (type(x), x, type(y), y) 176 return c
177 178 is_root = property(lambda self: self._id == 0) 179 parent = property(lambda self: self._priority_queue._nodes[(self._id - 1) / 2]) 180 left = property(lambda self: self._priority_queue._nodes[self._id * 2 + 1]) 181 right = property(lambda self: self._priority_queue._nodes[self._id * 2 + 2]) 182
183 - def promote(self):
184 assert False, type(self)
185
186 - def node_type(self):
187 assert False
188
189 -class _InteriorNode(_Node):
190 191 _content = None 192
193 - def __init__(self, priority_queue):
194 _Node.__init__(self, priority_queue) 195 self._content = MINUS_INFINITY
196
197 - def node_type(self):
198 return 'interior'
199
200 - def promote(self):
201 key = self._key 202 c = self._compare(self.left.content, self.right.content) 203 if c <= 0: 204 self.content = self.left.content 205 self.left.promote() 206 else: 207 self.content = self.right.content 208 self.right.promote()
209
210 - def _get_content(self):
211 return self._content
212
213 - def _set_content(self, content):
214 self._content = content
215 216 content = property(_get_content, _set_content)
217
218 -class _InputNode(_Node):
219 220 _source = None # int 221 _done = None 222 _buffer = None # _Buffer 223
224 - def __init__(self, priority_queue, source):
225 _Node.__init__(self, priority_queue) 226 self._done = False 227 self._source = source 228 self._buffer = _Buffer(self)
229
230 - def __repr__(self):
231 # reporting content for input nodes is dangerous 232 return '%s#%s(?)' % (self.node_type(), self._id)
233
234 - def node_type(self):
235 return 'input'
236
237 - def add(self, input):
238 if self._done: 239 raise PriorityQueueInputClosedException(self._source) 240 self._buffer.add(input)
241
242 - def done(self):
243 self._done = True
244
245 - def promote(self):
246 self._buffer.next()
247 248 content = property(lambda self: self._buffer.current()) 249 source = property(lambda self: self._source)
250
251 -class _FillerNode(_Node):
252
253 - def __init__(self, priority_queue):
254 _Node.__init__(self, priority_queue)
255
256 - def node_type(self):
257 return 'filler'
258 259 content = property(lambda self: INFINITY)
260
261 -class _Buffer(object):
262 263 _lock = None 264 _input_node = None 265 _read = None # list of objects 266 _read_position = None # position of current item in _read 267 _write = None # list of objects 268 _last_input = None 269 _done = None 270
271 - def __init__(self, input_node):
272 self._lock = threading.Condition() 273 self._input_node = input_node 274 self._read = [] 275 self._read_position = 0 276 self._write = [] 277 self._last_input = None 278 self._done = False
279 # trace('%s: created' % self) 280
281 - def __repr__(self):
282 buffer = ['buffer<'] 283 buffer.append(str(id(self))) 284 buffer.append('>#') 285 buffer.append(str(self._input_node.source)) 286 buffer.append('(pq:') 287 buffer.append(str(self._priority_queue)) 288 buffer.append(', read:') 289 buffer.append(_describe_list(self._read)) 290 buffer.append(', write: ') 291 buffer.append(_describe_list(self._write)) 292 buffer.append(')') 293 return ''.join(buffer)
294
295 - def current(self):
296 if self._done: 297 current = INFINITY 298 else: 299 self._ensure_read_not_empty() 300 current = self._read[self._read_position] 301 if current is INFINITY: 302 self._done = True 303 return current
304
305 - def _write_blocked(self):
306 return len(self._write) == WRITE_BUFFER_CAPACITY
307
308 - def _read_blocked(self):
309 write_buffer_size = len(self._write) 310 return (write_buffer_size < WRITE_BUFFER_CAPACITY and 311 (write_buffer_size == 0 or self._last_input is not INFINITY))
312
313 - def _ensure_read_not_empty(self):
314 if self._read_position == len(self._read): 315 self._lock.acquire() 316 while self._read_blocked(): 317 # trace('%s: read blocked' % self) 318 self._lock.wait(LOCK_WAIT_TIME) 319 # trace('%s: read unblocked' % self) 320 self._read = self._write 321 self._read_position = 0 322 self._write = [] 323 self._lock.notify() 324 self._lock.release()
325 326 _priority_queue = property(lambda self: self._input_node._priority_queue) 327
328 - def add(self, input):
329 # trace('%s: add %s' % (self, input)) 330 if (self._last_input is not None) and (self._input_node._compare(input, self._last_input) < 0): 331 raise PriorityQueueInputOrderingException(self._input_node, input, self._last_input) 332 if self._write_blocked(): 333 self._lock.acquire() 334 # If reader is blocking, then release it now. No point waiting for the write 335 # to be able to proceed. 336 self._lock.notify() 337 while self._write_blocked(): 338 # trace('%s: write blocked' % self) 339 self._lock.wait(LOCK_WAIT_TIME) 340 # trace('%s: write unblocked' % self) 341 self._lock.notify() 342 self._lock.release() 343 self._write.append(input) 344 self._last_input = input 345 if input is INFINITY: 346 # Don't keep reader waiting. This is like the notification to the reader when the writer 347 # is blocked. This case needs special handling because the buffer probably won't be full 348 # once INFINITY has arrived. 349 self._lock.acquire() 350 self._lock.notify() 351 self._lock.release()
352
353 - def next(self):
354 if self._done: 355 output = INFINITY 356 else: 357 self._ensure_read_not_empty() 358 output = self._read[self._read_position] 359 self._read[self._read_position] = None 360 self._read_position += 1 361 return output
362
363 -class PriorityQueueInputClosedException(Exception):
364
365 - def __init__(self, source):
366 Exception.__init__( 367 self, 368 'Attempt to call add or done on input stream %s after calling done' % source)
369
370 -class PriorityQueueInputOrderingException(Exception):
371
372 - def __init__(self, source, input, last_input):
373 Exception.__init__( 374 self, 375 'Incorrectly ordered input for input stream %s: %s after %s' % 376 (source, input, last_input))
377