maildir-scan.cc - bincimap

maildir-scan.cc (15855B)
      1 /* -*- Mode: c++; -*- */
      2 /*  --------------------------------------------------------------------
      3  *  Filename:
      4  *    maildir-scan.cc
      5  *  
      6  *  Description:
      7  *    Implementation of the Maildir class.
      8  *
      9  *  Authors:
     10  *    Andreas Aardal Hanssen <andreas-binc curly bincimap spot org>
     11  *
     12  *  Bugs:
     13  *
     14  *  ChangeLog:
     15  *
     16  *  --------------------------------------------------------------------
     17  *  Copyright 2002-2005 Andreas Aardal Hanssen
     18  *
     19  *  This program is free software; you can redistribute it and/or modify
     20  *  it under the terms of the GNU General Public License as published by
     21  *  the Free Software Foundation; either version 2 of the License, or
     22  *  (at your option) any later version.
     23  *
     24  *  This program is distributed in the hope that it will be useful,
     25  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
     26  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     27  *  GNU General Public License for more details.
     28  *
     29  *  You should have received a copy of the GNU General Public License
     30  *  along with this program; if not, write to the Free Software
     31  *  Foundation, Inc., 59 Temple Street #330, Boston, MA 02111-1307, USA.
     32  *  --------------------------------------------------------------------
     33  */
     34 #ifdef HAVE_CONFIG_H
     35 #include <config.h>
     36 #endif
     37 
     38 #include <fcntl.h>
     39 #include <dirent.h>
     40 #include <sys/stat.h>
     41 #include <unistd.h>
     42 
     43 #include "io.h"
     44 #include "maildir.h"
     45 #include "storage.h"
     46 
     47 using namespace Binc;
     48 using namespace ::std;
     49 
     50 namespace {
     51 
     52   //----------------------------------------------------------------------
     53   class Lock {
     54     string lock;
     55 
     56   public:
     57 
     58     //--
     59     Lock(const string &path)
     60     {
     61       IO &logger = IOFactory::getInstance().get(2);
     62 
     63       lock = (path == "" ? "." : path) + "/bincimap-scan-lock";
     64 
     65       int lockfd = -1;
     66       while ((lockfd = ::open(lock.c_str(),
     67 			      O_CREAT | O_WRONLY | O_EXCL, 0666)) == -1) {
     68 	if (errno != EEXIST) {
     69 	  logger << "unable to lock mailbox: " << lock
     70 		 << ", " << string(strerror(errno)) << endl;
     71 	  return;
     72 	}
     73 				 
     74 	struct stat mystat;
     75 	if (lstat(lock.c_str(), &mystat) == 0) {
     76 	  if ((time(0) - mystat.st_ctime) > 300) {
     77 	    if (unlink(lock.c_str()) == 0) {
     78               logger << "5 minute old lock detected: " << lock
     79                      << ", lock deleted." << endl;
     80               continue;
     81 	    } else {
     82               logger << "failed to force mailbox lock: " << lock
     83                      << ", " << string(strerror(errno)) << endl;
     84             }
     85 	  }
     86 	} else {
     87 	  if (errno != ENOENT) {
     88 	    string err = "invalid lock " + lock + ": "
     89 	      + strerror(errno);
     90 	    logger << err << endl;
     91 	    return;
     92 	  }
     93 	}
     94 	
     95 	// sleep one second.
     96 	sleep(1);
     97       }
     98 
     99       close(lockfd);
    100     }
    101 
    102     //--
    103     ~Lock()
    104     {
    105       IO &logger = IOFactory::getInstance().get(2);
    106 
    107       // remove the lock
    108       if (unlink(lock.c_str()) != 0)
    109 	logger << "failed to unlock mailbox: " << lock << ", "
    110 	       << strerror(errno) << endl;
    111     }
    112   };
    113 }
    114 
    115 //------------------------------------------------------------------------
    116 // scan the maildir. update flags, find messages in new/ and move them
    117 // to cur, setting the recent flag in memory only. check for expunged
    118 // messages. give newly arrived messages uids.
    119 //------------------------------------------------------------------------
    120 Maildir::ScanResult Maildir::scan(bool forceScan)
    121 {
    122   IO &logger = IOFactory::getInstance().get(2);
    123 
    124   const string newpath = path + "/new/";
    125   const string curpath = path + "/cur/";
    126   const string uidvalfilename = path + "/bincimap-uidvalidity";
    127   const string cachefilename = path + "/bincimap-cache";
    128 
    129   // check wether or not we need to bother scanning the folder.
    130   if (firstscan || forceScan) {
    131     struct stat oldstat;
    132     if (stat(newpath.c_str(), &oldstat) != 0) {
    133       setLastError("Invalid Mailbox, " + newpath + ": "
    134 		   + string(strerror(errno)));
    135       return PermanentError;
    136     }
    137 
    138     old_new_st_mtime = oldstat.st_mtime;
    139     old_new_st_ctime = oldstat.st_ctime;
    140 
    141     if (stat(curpath.c_str(), &oldstat) != 0) {
    142       setLastError("Invalid Mailbox, " + curpath + ": "
    143 		   + string(strerror(errno)));
    144       return PermanentError;
    145     }
    146 
    147     old_cur_st_mtime = oldstat.st_mtime;
    148     old_cur_st_ctime = oldstat.st_ctime;
    149 
    150   } else {
    151     struct stat oldcurstat;
    152     struct stat oldnewstat;
    153     if (stat(newpath.c_str(), &oldnewstat) != 0) {
    154       setLastError("Invalid Mailbox, " + newpath + ": "
    155 		   + string(strerror(errno)));
    156       return PermanentError;
    157     }
    158 
    159     if (stat(curpath.c_str(), &oldcurstat) != 0) {
    160       setLastError("Invalid Mailbox, " + curpath + ": "
    161 		   + string(strerror(errno)));
    162       return PermanentError;
    163     }
    164 
    165     if (oldnewstat.st_mtime == old_new_st_mtime
    166 	&& oldnewstat.st_ctime == old_new_st_ctime
    167 	&& oldcurstat.st_mtime == old_cur_st_mtime
    168 	&& oldcurstat.st_ctime == old_cur_st_ctime)
    169       return Success;
    170 
    171     old_cur_st_mtime = oldcurstat.st_mtime;
    172     old_cur_st_ctime = oldcurstat.st_ctime;
    173     old_new_st_mtime = oldnewstat.st_mtime;
    174     old_new_st_ctime = oldnewstat.st_ctime;
    175   }
    176 
    177   // lock the directory as we are scanning. this prevents race
    178   // conditions with uid delegation
    179   Lock lock(path);
    180 
    181   // Read the cache file if it's there. It holds important information
    182   // about the state of the depository, and serves to communicate
    183   // changes to the depot across Binc IMAP instances that can not be
    184   // communicated via the depot itself.
    185   switch (readCache()) {
    186   case NoCache:
    187   case Error:
    188     // An error with reading the cache files when it's not the first
    189     // time we scan the depot is treated as an error.
    190     if (!firstscan && !readOnly) {
    191       old_cur_st_mtime = (time_t) 0;
    192       old_cur_st_ctime = (time_t) 0;
    193       old_new_st_mtime = (time_t) 0;
    194       old_new_st_ctime = (time_t) 0;
    195       return TemporaryError;
    196     }
    197 
    198     uidnextchanged = true;
    199     mailboxchanged = true;
    200     break;
    201   default:
    202     break;
    203   }
    204 
    205   // open new/ directory
    206   DIR *pdir = opendir(newpath.c_str());
    207   if (pdir == 0) {
    208     string reason = "failed to open \"" + newpath + "\" (";
    209     reason += strerror(errno);
    210     reason += ")";
    211     setLastError(reason);
    212 
    213     return PermanentError;
    214   }
    215 
    216   // scan all entries
    217   struct dirent *pdirent;
    218   while ((pdirent = readdir(pdir)) != 0) {
    219     // "Unless you're writing messages to a maildir, the format of a
    220     // unique name is none of your business. A unique name can be
    221     // anything that doesn't contain a colon (or slash) and doesn't
    222     // start with a dot. Do not try to extract information from unique
    223     // names." - The Maildir spec from cr.yp.to
    224     string filename = pdirent->d_name;
    225     if (filename[0] == '.'
    226 	|| filename.find(':') != string::npos
    227 	|| filename.find('/') != string::npos)
    228       continue;
    229 
    230     string fullfilename = newpath + filename;
    231 
    232     // We need to find the timestamp of the message in order to
    233     // determine whether or not it's safe to move the message in from
    234     // new/. qmail's default message file naming algorithm forces us
    235     // to never move messages out of new/ that are less than one
    236     // second old.
    237     struct stat mystat;
    238     if (stat(fullfilename.c_str(), &mystat) != 0) {
    239       if (errno == ENOENT) {
    240         // prevent looping due to stale symlinks
    241         if (lstat(fullfilename.c_str(), &mystat) == 0) { 
    242           logger << "dangling symlink: " << fullfilename << endl; 
    243           continue;
    244         }
    245 
    246 	// a rare race between readdir and stat force us to restart
    247 	// the scan.
    248 	closedir(pdir);
    249 	
    250 	if ((pdir = opendir(newpath.c_str())) == 0) {
    251 	  string reason = "Warning: opendir(\"" + newpath + "\") == 0 (";
    252 	  reason += strerror(errno);
    253 	  reason += ")";
    254 	  setLastError(reason);
    255 	  
    256 	  return PermanentError;
    257 	}
    258       } else
    259 	logger << "junk in Maildir: \"" << fullfilename << "\": "
    260 	       << strerror(errno);
    261 
    262       continue;
    263     }
    264 
    265     // this is important. do not move messages from new/ that are not
    266     // at least one second old or messages may disappear. this
    267     // introduces a special case: we can not cache the old st_ctime
    268     // and st_mtime. the next time the mailbox is scanned, it must not
    269     // simply be skipped. :-)
    270 
    271     vector<MaildirMessage>::const_iterator newIt = newMessages.begin();
    272     bool ours = false;
    273     for (; newIt != newMessages.end(); ++newIt) {
    274       if ((filename == (*newIt).getUnique())
    275 	  && ((*newIt).getInternalFlags() & MaildirMessage::Committed)) {
    276 	ours = true;
    277 	break;
    278       }
    279     }
    280     
    281     if (!ours && ::time(0) <= mystat.st_mtime) {
    282       old_cur_st_mtime = (time_t) 0;
    283       old_cur_st_ctime = (time_t) 0;
    284       old_new_st_mtime = (time_t) 0;
    285       old_new_st_ctime = (time_t) 0;
    286       continue;
    287     }
    288 
    289     // move files from new/ to cur/
    290     string newName = curpath + pdirent->d_name;
    291     if (rename((newpath + pdirent->d_name).c_str(), 
    292 	       (newName + ":2,").c_str()) != 0) {
    293       logger << "error moving messages from"
    294 	" new to cur: skipping " << newpath 
    295 	     << pdirent->d_name << ": " << strerror(errno) << endl;
    296       continue;
    297     }
    298   }
    299 
    300   closedir(pdir);
    301 
    302   // Now, assume all known messages were expunged and have them prove
    303   // otherwise.
    304   {
    305     Mailbox::iterator i = begin(SequenceSet::all(), INCLUDE_EXPUNGED | SQNR_MODE);
    306     for (; i != end(); ++i)
    307       (*i).setExpunged();
    308   }
    309 
    310   // Then, scan cur
    311   // open directory
    312 
    313   if ((pdir = opendir(curpath.c_str())) == 0) {
    314     string reason = "Maildir::scan::opendir(\"" + curpath + "\") == 0 (";
    315     reason += strerror(errno);
    316     reason += ")";
    317 
    318     setLastError(reason);
    319     return PermanentError;
    320   }
    321 
    322   // erase all old maps between fixed filenames and actual file names.
    323   // we'll get a new list now, which will be more up to date.
    324   index.clearFileNames();
    325 
    326   // this is to sort recent messages by internaldate
    327   multimap<unsigned int, MaildirMessage> tempMessageMap;
    328 
    329   // scan all entries
    330   while ((pdirent = readdir(pdir)) != 0) {
    331     string filename = pdirent->d_name;
    332     if (filename[0] == '.')
    333       continue;
    334 
    335     string uniquename;
    336     string standard;
    337     string::size_type pos;
    338     if ((pos = filename.find(':')) != string::npos) {
    339       uniquename = filename.substr(0, pos);
    340 
    341       string tmp = filename.substr(pos);
    342       if ((pos = tmp.find("2,")) != string::npos)
    343 	standard = tmp.substr(pos + 2);
    344 
    345     } else
    346       uniquename = filename;
    347 
    348     unsigned char mflags = Message::F_NONE;
    349     for (string::const_iterator i = standard.begin();
    350 	 i != standard.end(); ++i) {
    351       switch (*i) {
    352       case 'R': mflags |= Message::F_ANSWERED; break;
    353       case 'S': mflags |= Message::F_SEEN; break;
    354       case 'T': mflags |= Message::F_DELETED; break;
    355       case 'D': mflags |= Message::F_DRAFT; break;
    356       case 'F': mflags |= Message::F_FLAGGED; break;
    357       default: break;
    358       }
    359     }
    360 
    361     struct stat mystat;
    362     MaildirMessage *message = get(uniquename);
    363     if (!message || message->getInternalDate() == 0) {
    364       string fullfilename = curpath + filename;
    365       if (stat(fullfilename.c_str(), &mystat) != 0) {
    366 	if (errno == ENOENT) {
    367           // prevent looping due to stale symlinks
    368           if (lstat(fullfilename.c_str(), &mystat) == 0) { 
    369             logger << "dangling symlink: " << fullfilename << endl; 
    370             continue;
    371           }
    372 	  // a rare race between readdir and stat force us to restart
    373 	  // the scan.
    374 	  index.clearFileNames();
    375 	  
    376 	  closedir(pdir);
    377 	  
    378 	  if ((pdir = opendir(newpath.c_str())) == 0) {
    379 	    string reason = "Warning: opendir(\"" + newpath + "\") == 0 (";
    380 	    reason += strerror(errno);
    381 	    reason += ")";
    382 	    setLastError(reason);
    383 	    
    384 	    return PermanentError;
    385 	  }
    386 	}
    387 	
    388 	continue;
    389       }
    390 
    391       mailboxchanged = true;
    392     }
    393 
    394     index.insert(uniquename, 0, filename);
    395   
    396     // If we have this message in memory already..
    397     if (message) {
    398 
    399       if (message->getInternalDate() == 0) {
    400 	mailboxchanged = true;
    401 	message->setInternalDate(mystat.st_mtime);
    402       }
    403 
    404       // then confirm that this message was not expunged
    405       message->setUnExpunged();
    406 
    407       // update the flags with what new flags we found in the filename,
    408       // but keep the \Recent flag regardless.
    409       if (mflags != (message->getStdFlags() & ~Message::F_RECENT)) {
    410 	int oldflags = message->getStdFlags();
    411 	message->resetStdFlags();
    412 	message->setStdFlag(mflags | (oldflags & Message::F_RECENT));
    413       }
    414 
    415       continue;
    416     }
    417 
    418     // Wait with delegating UIDs until all entries have been
    419     // read. Only then can we sort by internaldate and delegate new
    420     // UIDs in the proper order.
    421     MaildirMessage m(*this);
    422     m.setUID(0);
    423     m.setSize(0);
    424     m.setInternalDate(mystat.st_mtime);
    425     m.setStdFlag((mflags | Message::F_RECENT) & ~Message::F_EXPUNGED);
    426     m.setUnique(uniquename);
    427     tempMessageMap.insert(make_pair((unsigned int) mystat.st_mtime, m));
    428 
    429     mailboxchanged = true;
    430   }
    431 
    432   closedir(pdir);
    433 
    434   // Recent messages are added, ordered by internaldate. Also delegate
    435   // UIDs.
    436   {
    437     int readonlyuidnext = uidnext;
    438     multimap<unsigned int, MaildirMessage>::iterator i = tempMessageMap.begin();
    439     while (i != tempMessageMap.end()) {
    440       i->second.setUID(readOnly ? readonlyuidnext++ : uidnext++);
    441       multimap<unsigned int, MaildirMessage>::iterator itmp = i;
    442       ++itmp;
    443       add(i->second);
    444       tempMessageMap.erase(i);
    445       i = itmp;
    446       uidnextchanged = true;
    447     }
    448   }
    449 
    450   tempMessageMap.clear();
    451 
    452   // Some messages may have been detected by another server, and then
    453   // written to the cache, and then expunged again without us
    454   // accessing the cache nor the maildir. We need to remove those from
    455   // our internal message list.
    456   Mailbox::iterator jj = begin(SequenceSet::all(), INCLUDE_EXPUNGED | SQNR_MODE);
    457   while (jj != end()) {
    458     MaildirMessage &message = (MaildirMessage &)*jj;
    459 
    460     if (message.isExpunged()) {
    461       if (message.getInternalFlags() & MaildirMessage::JustArrived) {
    462 	jj.erase();
    463 	continue;
    464       }
    465     } else if (message.getInternalFlags() & MaildirMessage::JustArrived) {
    466       message.clearInternalFlag(MaildirMessage::JustArrived);
    467     }
    468 
    469     ++jj;
    470   }
    471 
    472   // Special case: The first time we scan is in SELECT. All flags
    473   // changes for new messages will then appear to be recent, and
    474   // to avoid this to be sent to the client as a pending update,
    475   // we explicitly unset the "flagsChanged" flag in all messages.
    476   if (firstscan) {
    477     unsigned int lastuid = 0;
    478 
    479     Mailbox::iterator ii
    480       = begin(SequenceSet::all(), INCLUDE_EXPUNGED | SQNR_MODE);
    481     for (; ii != end(); ++ii) {
    482       MaildirMessage &message = (MaildirMessage &)*ii;
    483       message.clearInternalFlag(MaildirMessage::JustArrived);
    484 
    485       if (lastuid < message.getUID())
    486 	lastuid = message.getUID();
    487       else {
    488 	logger << "UID values are not strictly ascending in this"
    489 	  " mailbox: " << path << ". This is usually caused by "
    490 	       << "access from a broken accessor. Bumping UIDVALIDITY." 
    491 	       << endl;
    492 
    493 	setLastError("An error occurred while scanning the mailbox. "
    494 		     "Please contact your system administrator.");
    495 
    496 	if (!readOnly) {
    497 	  bumpUidValidity(path);
    498 
    499 	  old_cur_st_mtime = (time_t) 0;
    500 	  old_cur_st_ctime = (time_t) 0;
    501 	  old_new_st_mtime = (time_t) 0;
    502 	  old_new_st_ctime = (time_t) 0;
    503 	  return TemporaryError;
    504 	} else {
    505       	  return PermanentError;
    506 	}
    507       }
    508       
    509       message.setFlagsUnchanged();
    510     }
    511   }
    512 
    513   if (mailboxchanged && !readOnly) {
    514     if (!writeCache())
    515       return PermanentError;
    516 
    517     mailboxchanged = false;
    518   }
    519 
    520   if (uidnextchanged && !readOnly) {
    521     Storage uidvalfile(uidvalfilename, Storage::WriteOnly);
    522     uidvalfile.put("depot", "_uidvalidity", toString(uidvalidity));
    523     uidvalfile.put("depot", "_uidnext", toString(uidnext));
    524     uidvalfile.put("depot", "_version", UIDVALFILEVERSION);
    525     if (!uidvalfile.commit()) {
    526       setLastError("Unable to save cache file.");
    527       return PermanentError;
    528     }
    529 
    530     uidnextchanged = false;
    531   }
    532 
    533   firstscan = false;
    534   newMessages.clear();
    535   return Success;
    536 }
	bincimap
	Log \| Files \| Refs \| LICENSE