PYTHON   17

DeDuper

Guest on 22nd September 2022 12:18:52 AM

  1. import os
  2. import errno
  3. import time
  4. import cPickle
  5.  
  6. from Mailman.Errors import DiscardMessage
  7. from Mailman.Logging.Syslog import syslog
  8.  
  9. def process(mlist, msg, msgdata):
  10.     msgid = msg.get('message-id', None)
  11.     if not msgid:
  12.         return
  13.     filename = os.path.join(mlist.fullpath(), 'message-ids.pck')
  14.     try:
  15.         fp = open(filename, 'r')
  16.         pairs = cPickle.load(fp)
  17.         if pairs.has_key(msgid):
  18.             syslog('vette', '%s post from %s with message-id=%s is a duplicate',
  19.                     mlist.internal_name(), msg.get_sender(), msgid)
  20.             raise DiscardMessage
  21.         else:
  22.             pairs[msgid] = time.time()
  23.             remove = [k for k,v in pairs.iteritems() if pairs[msgid]-v > 300]
  24.             for k in remove: del pairs[k]
  25.             fp = open(filename, 'w')
  26.             cPickle.dump(pairs, fp)
  27.     except IOError, e:
  28.         if e.errno == errno.ENOENT:
  29.             fp = open(filename, 'w')
  30.             pairs = {}
  31.             pairs[msgid] = time.time()
  32.             cPickle.dump(pairs, fp)
  33.     finally:
  34.         fp.close()

Raw Paste


Login or Register to edit or fork this paste. It's free.