diff --git a/.gitignore b/.gitignore index 9310832..f82657d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ build dist .tox +*.swp diff --git a/pygtail/core.py b/pygtail/core.py index aff3f42..c5df66a 100755 --- a/pygtail/core.py +++ b/pygtail/core.py @@ -23,14 +23,17 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA from __future__ import print_function -from os import stat -from os.path import exists, getsize +from stat import S_IFREG +from os import stat, listdir, walk +from os.path import exists, getsize, basename +from os.path import join as path_join import sys import glob import gzip from optparse import OptionParser -__version__ = '0.6.0' + +__version__ = '0.6.1' PY3 = sys.version_info[0] == 3 @@ -61,7 +64,7 @@ class Pygtail(object): copytruncate Support copytruncate-style log rotation (default: True) """ def __init__(self, filename, offset_file=None, paranoid=False, copytruncate=True, - every_n=0, on_update=False): + every_n=0, on_update=False, use_update_file=True): self.filename = filename self.paranoid = paranoid self.every_n = every_n @@ -73,6 +76,7 @@ def __init__(self, filename, offset_file=None, paranoid=False, copytruncate=True self._since_update = 0 self._fh = None self._rotated_logfile = None + self._use_update_file = use_update_file # if offset file exists and non-empty, open and parse it if exists(self._offset_file) and getsize(self._offset_file): @@ -181,11 +185,14 @@ def _update_offset_file(self): """ if self.on_update: self.on_update() - offset = self._filehandle().tell() - inode = stat(self.filename).st_ino - fh = open(self._offset_file, "w") - fh.write("%s\n%s\n" % (inode, offset)) - fh.close() + + if self._use_update_file: + offset = self._filehandle().tell() + inode = stat(self.filename).st_ino + fh = open(self._offset_file, "w") + fh.write("%s\n%s\n" % (inode, offset)) + fh.close() + self._since_update = 0 def _determine_rotated_logfile(self): @@ -249,12 +256,83 @@ def _check_rotated_filename_candidates(self): return None def _get_next_line(self): - line = self._filehandle().readline() + # flush handle before reading; this helps emulate `tail -f` behavior. + try: + self._filehandle().flush() + line = self._filehandle().readline() + except IOError: + # In case the referred path no longer exists + raise StopIteration if not line: raise StopIteration self._since_update += 1 return line +class Pygdir(object): + """ + Create a monitor object for a specified directory. + + Every time the iterator object is queried, the file set is also updated. + + base_dir The directory to monitor + + Other parameters are passed as-is into Pygtail. + """ + def __init__(self, base_dir, filter_func=None, base_dir_glob=None, paranoid=False, copytruncate=True, + every_n=0, on_update=False): + self._base_dir = base_dir + self._base_dir_glob = base_dir_glob + self._file_set = {} + self._paranoid = paranoid + self._copytruncate = copytruncate + self._every_n = every_n + self._on_update = on_update + self._filter_func = filter_func + + def _make_filename(self, filename): + return path_join(self._base_dir, filename) + + def _get_all_base_dir_files(self): + if self._base_dir_glob is not None: + return [y for x in walk(self._base_dir) for y in glob.glob(path_join(x[0], self._base_dir_glob))] + else: + return listdir(self._base_dir) + + def _regular_files_in_dir(self): + return filter(self._filter_func, [f for f in self._get_all_base_dir_files() if exists(f) and stat(self._make_filename(f)).st_mode & S_IFREG > 0]) + + def _update_file_set(self): + """ + Update the file set to be monitored: all new files are added, existing files aren't touched. + + XXX: This could mean that if an I/O error occurs (e.g., a monitored file is deleted, and then re-created) we would stop monitoring that file. + """ + def make_pygtail(filename): + return Pygtail(self._make_filename(filename), None, self._paranoid, self._copytruncate, self._every_n, self._on_update, False) + + self._file_set.update(dict((f,make_pygtail(f)) for f in self._regular_files_in_dir() if f not in self._file_set.keys())) + + def __next__(self): + return self.next() + + def next(self): + + # XXX: consider moving this to after iteration; starvation could occur either way. + # Either super-active old files will choke out new files, or many-created files will choke out old files. Currently, the code errs on side of few creations and many additions + self._update_file_set() + for p in self._file_set.values(): + try: + return p.__next__() + except StopIteration: + # eat the exception and continue trying to find + # an entry with a next line + pass + + # we're out of files, so really no more lines + raise StopIteration + + def __iter__(self): + return self def main(): # command-line parsing