fabio.fabioutils

Package fabio :: Module fabioutils |
[hide private]
[frames] | no frames]

# Source Code for Module fabio.fabioutils

      1  #!/usr/bin/env python
      2  #coding: utf8
      3
      4  """
      5  General purpose utilities functions for fabio
      6  """
      7  from __future__ import with_statement
      8  import re, os, logging, threading, sys
      9  import StringIO as stringIO
     10  logger = logging.getLogger("fabioutils")
     11  from compression import bz2, gzip
     12  import traceback
     13
     14
     15
     16  FILETYPES = {
     17      # extension NNNimage fabioclass
     18      # type consistency - always use a list if one case is
     19      'edf'    : ['edf'],
     20      'cor'    : ['edf'],
     21      'pnm'    : ['pnm'],
     22      'pgm'    : ['pnm'],
     23      'pbm'    : ['pnm'],
     24      'tif'    : ['tif'],
     25      'tiff'   : ['tif'],
     26      'img'    : ['adsc', 'OXD', 'HiPiC'],
     27      'mccd'   : ['marccd'],
     28      'mar2300': ['mar345'],
     29      'sfrm'   : ['bruker100'],
     30      'msk'    : ['fit2dmask'],
     31      'spr'    : ['fit2dspreadsheet'],
     32      'dm3'    : ['dm3'],
     33      'kcd'    : ['kcd'],
     34      'cbf'    : ['cbf'],
     35      'xml'    : ["xsd"],
     36      'xsd'    : ["xsd"],
     37               }
     38
     39  # Add bzipped and gzipped
     40  for key in FILETYPES.keys():
     41      FILETYPES[key + ".bz2"] = FILETYPES[key]
     42      FILETYPES[key + ".gz"] = FILETYPES[key]
     43
     44
     45  # Compressors
     46
     47  COMPRESSORS = {}
     48
     49
     50  dictAscii = {None:[chr(i) for i in range(32, 127)],
     51             }
     52
     53  try:
     54      lines = os.popen("gzip -h 2>&1").read()
     55      # Looking for "usage"
     56      if "sage" in lines:
     57          COMPRESSORS['.gz'] = 'gzip -dc '
     58      else:
     59          COMPRESSORS['.gz'] = None
     60  except Exception:
     61      COMPRESSORS['.gz'] = None
     62
     63  try:
     64      lines = os.popen("bzip2 -h 2>&1").read()
     65      # Looking for "usage"
     66      if "sage" in lines:
     67          COMPRESSORS['.bz2'] = 'bzip2 -dc '
     68      else:
     69          COMPRESSORS['.bz2'] = None
     70  except Exception:
     71      COMPRESSORS['.bz2'] = None
     72
     73 -def deprecated(func):
     74      """
     75      used to deprecate a function/method: prints a lot of warning messages to enforce the modifaction of the code
     76      """
     77      def wrapper(*arg, **kw):
     78          """
     79          decorator that deprecates the use of a function
     80          """
     81          logger.warning("%s is Deprecated !!! %s" % (func.func_name, os.linesep.join([""] + traceback.format_stack()[:-1])))
     82          return func(*arg, **kw)
     83      return wrapper
     84
     85
     86 -def getnum(name):
     87      """
     88      # try to figure out a file number
     89      # guess it starts at the back
     90      """
     91      stem , num, post_num = numstem(name)
     92      try:
     93          return int(num)
     94      except ValueError:
     95          return None
     96
     97 -class FilenameObject(object):
     98      """
     99      The 'meaning' of a filename ...
    100      """
    101 -    def __init__(self, stem=None,
    102              num=None,
    103              directory=None,
    104              format=None,
    105              extension=None,
    106              postnum=None,
    107              digits=4,
    108              filename = None):
    109          """
    110          This class can either be instanciated by a set of parameters like  directory, prefix, num, extension, ...
    111
    112          @param stem: the stem is a kind of prefix (str)
    113          @param num: image number in the serie (int)
    114          @param directory: name of the directory (str)
    115          @param format: ??
    116          @param extension:
    117          @param postnum:
    118          @param digits: Number of digits used to print num
    119
    120          Alternative constructor:
    121
    122          @param filename: fullpath of an image file to be deconstructed into directory, prefix, num, extension, ...
    123
    124          """
    125
    126
    127          self.stem = stem
    128          self.num = num
    129          self.format = format
    130          self.extension = extension
    131          self.digits = digits
    132          self.postnum = postnum
    133          self.directory = directory
    134          self.compressed = None
    135          if filename is not None:
    136              self.deconstruct_filename(filename)
    137
    138
    139 -    def str(self):
    140          """ Return a string representation """
    141          fmt = "stem %s, num %s format %s extension %s " + \
    142                  "postnum = %s digits %s dir %s"
    143          return fmt % tuple([str(x) for x in [
    144                      self.stem ,
    145                      self.num ,
    146                      self.format ,
    147                      self.extension ,
    148                      self.postnum ,
    149                      self.digits ,
    150                      self.directory ] ])
    151      __repr__ = str
    152
    153 -    def tostring(self):
    154          """
    155          convert yourself to a string
    156          """
    157          name = self.stem
    158          if self.digits is not None and self.num is not None:
    159              fmt = "%0" + str(self.digits) + "d"
    160              name += fmt % self.num
    161          if self.postnum is not None:
    162              name += self.postnum
    163          if self.extension is not None:
    164              name += self.extension
    165          if self.directory is not None:
    166              name = os.path.join(self.directory, name)
    167          return name
    168
    169
    170 -    def deconstruct_filename(self, filename):
    171          """
    172          Break up a filename to get image type and number
    173          """
    174          direc, name = os.path.split(filename)
    175          direc = direc or None
    176          parts = name.split(".")
    177          compressed = False
    178          stem = parts[0]
    179          extn = ""
    180          postnum = ""
    181          ndigit = 4
    182          num = None
    183          typ = None
    184          if parts[-1] in ["gz", "bz2"]:
    185              extn = "." + parts[-1]
    186              parts = parts[:-1]
    187              compressed = True
    188          if parts[-1] in FILETYPES.keys():
    189              typ = FILETYPES[parts[-1]]
    190              extn = "." + parts[-1] + extn
    191              try:
    192                  stem, numstring, postnum = numstem(".".join(parts[:-1]))
    193                  num = int(numstring)
    194                  ndigit = len(numstring)
    195              except Exception, err:
    196                  # There is no number - hence make num be None, not 0
    197                  logger.debug("l176: %s" % err)
    198                  num = None
    199                  stem = "".join(parts[:-1])
    200          else:
    201              # Probably two type left
    202              if len(parts) == 1:
    203                  # Probably GE format stem_numb
    204                  parts2 = parts[0].split("_")
    205                  if parts2[-1].isdigit():
    206                      num = int(parts2[-1])
    207                      ndigit = len(parts2[-1])
    208                      typ = ['GE']
    209                      stem = "_".join(parts2[:-1]) + "_"
    210              else:
    211                  try:
    212                      num = int(parts[-1])
    213                      ndigit = len(parts[-1])
    214                      typ = ['bruker']
    215                      stem = ".".join(parts[:-1]) + "."
    216                  except Exception, err:
    217                      logger.debug("l196: %s" % err)
    218                      typ = None
    219                      extn = "." + parts[-1] + extn
    220                      numstring = ""
    221                      try:
    222                          stem , numstring, postnum = numstem(".".join(parts[:-1]))
    223                      except Exception, err:
    224                          logger.debug("l202: %s" % err)
    225                          raise
    226                      if numstring.isdigit():
    227                          num = int(numstring)
    228                          ndigit = len(numstring)
    229                  #            raise Exception("Cannot decode "+filename)
    230
    231          self.stem = stem
    232          self.num = num
    233          self.directory = direc
    234          self.format = typ
    235          self.extension = extn
    236          self.postnum = postnum
    237          self.digits = ndigit
    238          self.compressed = compressed
    239
    240 -def numstem(name):
    241      """ cant see how to do without reversing strings
    242      Match 1 or more digits going backwards from the end of the string
    243      """
    244      reg = re.compile(r"^(.*?)(-?[0-9]{0,9})(\D*)$")
    245      #reg = re.compile("""(\D*)(\d\d*)(\w*)""")
    246      try:
    247          res = reg.match(name).groups()
    248          #res = reg.match(name[::-1]).groups()
    249          #return [ r[::-1] for r in res[::-1]]
    250          if len(res[0]) == len(res[1]) == 0: # Hack for file without number
    251              return [res[2], '', '']
    252          return [ r for r in res]
    253      except AttributeError: # no digits found
    254          return [name, "", ""]
    255
    256  @deprecated
    257 -def deconstruct_filename(filename):
    258      """
    259      Function for backward compatibility.
    260      Deprecated
    261      """
    262      return FilenameObject(filename=filename)
    263
    264 -def construct_filename(filename, frame=None):
    265      "Try to construct the filename for a given frame"
    266      fobj = FilenameObject(filename=filename)
    267      if frame is not None:
    268          fobj.num = frame
    269      return fobj.tostring()
    270
    271 -def next_filename(name, padding=True):
    272      """ increment number """
    273      fobj = FilenameObject(filename=name)
    274      fobj.num += 1
    275      if not padding:
    276          fobj.digits = 0
    277      return fobj.tostring()
    278
    279 -def previous_filename(name, padding=True):
    280      """ decrement number """
    281      fobj = FilenameObject(filename=name)
    282      fobj.num -= 1
    283      if not padding:
    284          fobj.digits = 0
    285      return fobj.tostring()
    286
    287 -def jump_filename(name, num, padding=True):
    288      """ jump to number """
    289      fobj = FilenameObject(filename=name)
    290      fobj.num = num
    291      if not padding:
    292          fobj.digits = 0
    293      return fobj.tostring()
    294
    295
    296 -def extract_filenumber(name):
    297      """ extract file number """
    298      fobj = FilenameObject(filename=name)
    299      return fobj.num
    300
    301 -def isAscii(name, listExcluded=None):
    302      """
    303      @param name: string to check
    304      @param listExcluded: list of char or string excluded.
    305      @return: True of False whether  name is pure ascii or not
    306      """
    307      isascii = None
    308      try:
    309          name.decode("ascii")
    310      except UnicodeDecodeError:
    311          isascii = False
    312      else:
    313          if listExcluded:
    314              isascii = not(any(bad in  name for bad in listExcluded))
    315          else:
    316              isascii = True
    317      return isascii
    318
    319 -def toAscii(name, excluded=None):
    320      """
    321      @param name: string to check
    322      @param excluded: tuple of char or string excluded (not list: they are mutable).
    323      @return: the name with all non valid char removed
    324      """
    325      if excluded not in dictAscii:
    326          ascii = dictAscii[None][:]
    327          for i in excluded:
    328              if i in ascii:
    329                  ascii.remove(i)
    330              else:
    331                  logger.error("toAscii: % not in ascii table" % i)
    332          dictAscii[excluded] = ascii
    333      else:
    334          ascii = dictAscii[excluded]
    335      out = [i for i in str(name) if i in ascii]
    336      return "".join(out)
    337
    338 -def nice_int(s):
    339      """
    340      Workaround that int('1.0') raises an exception
    341
    342      @param s: string to be converted to integer
    343      """
    344      try:
    345          return int(s)
    346      except ValueError:
    347          return int(float(s))
    348
    349
    350 -class StringIO(stringIO.StringIO):
    351      """
    352      just an interface providing the name and mode property to a StringIO
    353
    354      BugFix for MacOSX mainly
    355      """
    356 -    def __init__(self, data, fname=None, mode="r"):
    357          stringIO.StringIO.__init__(self, data)
    358          self.closed = False
    359          if fname == None:
    360              self.name = "fabioStream"
    361          else:
    362              self.name = fname
    363          self.mode = mode
    364          self.lock = threading.Semaphore()
    365          self.__size = None
    366
    367 -    def getSize(self):
    368          if self.__size is None:
    369              logger.debug("Measuring size of %s" % self.name)
    370              with self.lock:
    371                  pos = self.tell()
    372                  self.seek(0, os.SEEK_END)
    373                  self.__size = self.tell()
    374                  self.seek(pos)
    375          return self.__size
    376 -    def setSize(self, size):
    377          self.__size = size
    378      size = property(getSize, setSize)
    379
    380 -class File(file):
    381      """
    382      wrapper for "file" with locking
    383      """
    384 -    def __init__(self, name, mode="rb", buffering=0):
    385          """file(name[, mode[, buffering]]) -> file object
    386
    387          Open a file.  The mode can be 'r', 'w' or 'a' for reading (default),
    388          writing or appending.  The file will be created if it doesn't exist
    389          when opened for writing or appending; it will be truncated when
    390          opened for writing.  Add a 'b' to the mode for binary files.
    391          Add a '+' to the mode to allow simultaneous reading and writing.
    392          If the buffering argument is given, 0 means unbuffered, 1 means line
    393          buffered, and larger numbers specify the buffer size.  The preferred way
    394          to open a file is with the builtin open() function.
    395          Add a 'U' to mode to open the file for input with universal newline
    396          support.  Any line ending in the input file will be seen as a '\n'
    397          in Python.  Also, a file so opened gains the attribute 'newlines';
    398          the value for this attribute is one of None (no newline read yet),
    399          '\r', '\n', '\r\n' or a tuple containing all the newline types seen.
    400
    401          'U' cannot be combined with 'w' or '+' mode.
    402          """
    403          file.__init__(self, name, mode, buffering)
    404          self.lock = threading.Semaphore()
    405          self.__size = None
    406 -    def getSize(self):
    407          if self.__size is None:
    408              logger.debug("Measuring size of %s" % self.name)
    409              with self.lock:
    410                  pos = self.tell()
    411                  self.seek(0, os.SEEK_END)
    412                  self.__size = self.tell()
    413                  self.seek(pos)
    414          return self.__size
    415 -    def setSize(self, size):
    416          self.__size = size
    417      size = property(getSize, setSize)
    418
    419 -class UnknownCompressedFile(File):
    420      """
    421      wrapper for "File" with locking
    422      """
    423 -    def __init__(self, name, mode="rb", buffering=0):
    424          logger.warning("No decompressor found for this type of file (are gzip anf bz2 installed ???")
    425          File.__init__(self, name, mode, buffering)
    426
    427  if gzip is None:
    428      GzipFile = UnknownCompressedFile
    429  else:
    430 -    class GzipFile(gzip.GzipFile):
    431          """
    432          Just a wrapper forgzip.GzipFile providing the correct seek capabilities for python 2.5
    433          """
    434 -        def __init__(self, filename=None, mode=None, compresslevel=9, fileobj=None):
    435              """
    436              Wrapper with locking for constructor for the GzipFile class.
    437
    438              At least one of fileobj and filename must be given a
    439              non-trivial value.
    440
    441              The new class instance is based on fileobj, which can be a regular
    442              file, a StringIO object, or any other object which simulates a file.
    443              It defaults to None, in which case filename is opened to provide
    444              a file object.
    445
    446              When fileobj is not None, the filename argument is only used to be
    447              included in the gzip file header, which may includes the original
    448              filename of the uncompressed file.  It defaults to the filename of
    449              fileobj, if discernible; otherwise, it defaults to the empty string,
    450              and in this case the original filename is not included in the header.
    451
    452              The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', or 'wb',
    453              depending on whether the file will be read or written.  The default
    454              is the mode of fileobj if discernible; otherwise, the default is 'rb'.
    455              Be aware that only the 'rb', 'ab', and 'wb' values should be used
    456              for cross-platform portability.
    457
    458              The compresslevel argument is an integer from 1 to 9 controlling the
    459              level of compression; 1 is fastest and produces the least compression,
    460              and 9 is slowest and produces the most compression.  The default is 9.
    461              """
    462              gzip.GzipFile.__init__(self, filename, mode, compresslevel, fileobj)
    463              self.lock = threading.Semaphore()
    464              self.__size = None
    465
    466
    467          if sys.version_info < (2, 7):
    468 -            def getSize(self):
    469                  if self.__size is None:
    470                      logger.debug("Measuring size of %s" % self.name)
    471                      with open(self.filename, "rb") as f:
    472                          f.seek(-4)
    473                          self.__size = numpy.fromstring(f.read(4), dtype=numpy.uint32)
    474                  return self.__size
    475 -            def setSize(self, value):
    476                  self.__size = value
    477              size = property(getSize, setSize)
    478              @property
    479 -            def closed(self):
    480                  return self.fileobj is None
    481
    482 -            def seek(self, offset, whence=os.SEEK_SET):
    483                  """
    484                  Move to new file position.
    485
    486                  Argument offset is a byte count.  Optional argument whence defaults to
    487                  0 (offset from start of file, offset should be >= 0); other values are 1
    488                  (move relative to current position, positive or negative), and 2 (move
    489                  relative to end of file, usually negative, although many platforms allow
    490                  seeking beyond the end of a file).  If the file is opened in text mode,
    491                  only offsets returned by tell() are legal.  Use of other offsets causes
    492                  undefined behavior.
    493
    494                  This is a wrapper for seek to ensure compatibility with old python 2.5
    495                  """
    496                  if whence == os.SEEK_SET:
    497                      gzip.GzipFile.seek(self, offset)
    498                  elif whence == os.SEEK_CUR:
    499                      gzip.GzipFile.seek(self, offset + self.tell())
    500                  elif whence == os.SEEK_END:
    501                      gzip.GzipFile.seek(self, -1)
    502                      gzip.GzipFile.seek(self, offset + self.tell())
    503
    504  if bz2 is None:
    505      BZ2File = UnknownCompressedFile
    506  else:
    507 -    class BZ2File(bz2.BZ2File):
    508          "Wrapper with lock"
    509 -        def __init__(self, name , mode='r', buffering=0, compresslevel=9):
    510              """
    511              BZ2File(name [, mode='r', buffering=0, compresslevel=9]) -> file object
    512
    513              Open a bz2 file. The mode can be 'r' or 'w', for reading (default) or
    514              writing. When opened for writing, the file will be created if it doesn't
    515              exist, and truncated otherwise. If the buffering argument is given, 0 means
    516              unbuffered, and larger numbers specify the buffer size. If compresslevel
    517              is given, must be a number between 1 and 9.
    518
    519              Add a 'U' to mode to open the file for input with universal newline
    520              support. Any line ending in the input file will be seen as a '\n' in
    521              Python. Also, a file so opened gains the attribute 'newlines'; the value
    522              for this attribute is one of None (no newline read yet), '\r', '\n',
    523              '\r\n' or a tuple containing all the newline types seen. Universal
    524              newlines are available only when reading.
    525              """
    526              bz2.BZ2File.__init__(self, name , mode, buffering, compresslevel)
    527              self.lock = threading.Semaphore()
    528              self.__size = None
    529 -        def getSize(self):
    530              if self.__size is None:
    531                  logger.debug("Measuring size of %s" % self.name)
    532                  with self.lock:
    533                      pos = self.tell()
    534                      all = self.read()
    535                      self.__size = self.tell()
    536                      self.seek(pos)
    537              return self.__size
    538 -        def setSize(self, value):
    539              self.__size = value
    540          size = property(getSize, setSize)
    541