Context:
python 2.7.3
ubuntu
Symptoms：
after the code run, logs can be dumped in the RSS memory with GDB. BTW. the problem does not exist in Python 2.7.10+, but I can not find any related ChangeLog.
python 2.7.3
ubuntu
Symptoms：
after the code run, logs can be dumped in the RSS memory with GDB. BTW. the problem does not exist in Python 2.7.10+, but I can not find any related ChangeLog.
# coding=utf-8
import os
import sys
import time
import Queue
import logging
import threading
import traceback
class Reader(threading.Thread):
def __init__(self, q):
super(Reader, self).__init__(name="Reader")
self.queue = q
def run(self):
while True:
log = "1542085197 2018-11-13<SP>12:59:57 0.524 NONE - 124.89.70.249 124.89.70.249 - www.test.com /beauty.jpg GET - - -<||>- 400 0 - Mozilla/4.0<SP>(compatible;<SP>MSIE<SP>6.0;<SP>Windows<SP>NT<SP>4.0;<SP>); zh-cn<||>- -<||>80<||>-<||>http - - - 23332 - 165 165 0 - -"
counter = 1
arr = []
items = log.split()
items[8] = '-'
items[9] = '/'
while True:
counter += 1
ip = "1.1.%s.%s" % ((counter/256) % 255, counter%255)
items[5] = ip
items[6] = ip
log1=" ".join(items)
arr.append(log1)
if len(arr) >= 3000:
try:
self.queue.put(arr, timeout=1)
print("add a msg into queue!")
except Queue.Full:
print( 'reader: queue is full')
arr = []
if counter > 2100000:
time.sleep(10000)
class Matcher(threading.Thread):
def __init__(self, q):
super(Matcher, self).__init__(name='Matcher')
self.queue = q
def run(self):
counter = 0
while True:
try:
logs = self.queue.get(timeout=3)
counter += len(logs)
for log in logs:
item = log.split()
except Queue.Empty:
print("queue empty! ")
print ("rec %s" % counter)
time.sleep(0.1)
class XXer(object):
def __init__(self):
q = Queue.Queue(2048)
self.reader = Reader(q)
self.matcher = Matcher(q)
self._worker_init(self.reader)
self._worker_init(self.matcher)
def _worker_init(self, w):
w.setDaemon(True)
w.start()
return w
def run(self):
while True:
time.sleep(1)
if __name__ == "__main__":
obj = XXer()
obj.run()
. 