pyffi.spells

1 """ 2 :mod:`pyffi.spells` --- High level file operations 3 ================================================== 4 5 .. note:: 6 7 This module is based on wz's NifTester module, although 8 nothing of wz's original code is left in this module. 9 10 A :term:`toaster`, implemented by subclasses of the abstract 11 :class:`Toaster` class, walks over all files in a folder, and applies 12 one or more transformations on each file. Such transformations are 13 called :term:`spell`\ s, and are implemented by subclasses of the 14 abstract :class:`Spell` class. 15 16 A :term:`spell` can also run independently of a :term:`toaster` and be 17 applied on a branch directly. The recommended way of doing this is via 18 the :meth:`Spell.recurse` method. 19 20 Supported spells 21 ---------------- 22 23 For format specific spells, refer to the corresponding module. 24 25 .. toctree:: 26 :maxdepth: 2 27 28 pyffi.spells.cgf 29 pyffi.spells.dae 30 pyffi.spells.dds 31 pyffi.spells.kfm 32 pyffi.spells.nif 33 pyffi.spells.tga 34 35 Some spells are applicable on every file format, and those are documented 36 here. 37 38 .. autoclass:: SpellApplyPatch 39 :show-inheritance: 40 :members: 41 42 Adding new spells 43 ----------------- 44 45 To create new spells, derive your custom spells from the :class:`Spell` 46 class, and include them in the :attr:`Toaster.SPELLS` attribute of your 47 toaster. 48 49 .. autoclass:: Spell 50 :show-inheritance: 51 :members: READONLY, SPELLNAME, data, stream, toaster, 52 __init__, recurse, _datainspect, datainspect, _branchinspect, 53 branchinspect, dataentry, dataexit, branchentry, 54 branchexit, toastentry, toastexit 55 56 Grouping spells together 57 ------------------------ 58 59 It is also possible to create composite spells, that is, spells that 60 simply execute other spells. The following functions and classes can 61 be used for this purpose. 62 63 .. autofunction:: SpellGroupParallel 64 65 .. autofunction:: SpellGroupSeries 66 67 .. autoclass:: SpellGroupBase 68 :show-inheritance: 69 :members: 70 :undoc-members: 71 72 73 .. autoclass:: SpellGroupParallelBase 74 :show-inheritance: 75 :members: 76 :undoc-members: 77 78 79 .. autoclass:: SpellGroupSeriesBase 80 :show-inheritance: 81 :members: 82 :undoc-members: 83 84 Creating toaster scripts 85 ------------------------ 86 87 To create a new toaster script, derive your toaster from the :class:`Toaster` 88 class, and set the :attr:`Toaster.FILEFORMAT` attribute of your toaster to 89 the file format class of the files it can toast. 90 91 .. autoclass:: Toaster 92 :show-inheritance: 93 :members: 94 :inherited-members: 95 :undoc-members: 96 97 """ 98 99 # -------------------------------------------------------------------------- 100 # ***** BEGIN LICENSE BLOCK ***** 101 # 102 # Copyright (c) 2007-2011, Python File Format Interface 103 # All rights reserved. 104 # 105 # Redistribution and use in source and binary forms, with or without 106 # modification, are permitted provided that the following conditions 107 # are met: 108 # 109 # * Redistributions of source code must retain the above copyright 110 # notice, this list of conditions and the following disclaimer. 111 # 112 # * Redistributions in binary form must reproduce the above 113 # copyright notice, this list of conditions and the following 114 # disclaimer in the documentation and/or other materials provided 115 # with the distribution. 116 # 117 # * Neither the name of the Python File Format Interface 118 # project nor the names of its contributors may be used to endorse 119 # or promote products derived from this software without specific 120 # prior written permission. 121 # 122 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 123 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 124 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 125 # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 126 # COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 127 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 128 # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 129 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 130 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 131 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN 132 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 133 # POSSIBILITY OF SUCH DAMAGE. 134 # 135 # ***** END LICENSE BLOCK ***** 136 # -------------------------------------------------------------------------- 137 138 from ConfigParser import ConfigParser 139 from copy import deepcopy 140 from cStringIO import StringIO 141 import gc 142 from itertools import izip 143 import logging # Logger 144 try: 145 import multiprocessing # Pool 146 except ImportError: 147 # < py26 148 multiprocessing = None 149 import optparse 150 import os # remove 151 import os.path # getsize, split, join 152 import re # for regex parsing (--skip, --only) 153 import shlex # shlex.split for parsing option lists in ini files 154 import subprocess 155 import sys # sys.stdout 156 import tempfile 157 158 import pyffi # for pyffi.__version__ 159 import pyffi.object_models # pyffi.object_models.FileFormat

160 161 -class Spell(object):

162 """Spell base class. A spell takes a data file and then does something 163 useful with it. The main entry point for spells is :meth:`recurse`, so if you 164 are writing new spells, start with reading the documentation with 165 :meth:`recurse`. 166 """ 167 168 data = None 169 """The :class:`~pyffi.object_models.FileFormat.Data` instance 170 this spell acts on.""" 171 172 stream = None 173 """The current ``file`` being processed.""" 174 175 toaster = None 176 """The :class:`Toaster` instance this spell is called from.""" 177 178 changed = False 179 """Whether the spell changed the data. If ``True``, the file will be 180 written back, otherwise not. 181 """ 182 183 # spells are readonly by default 184 READONLY = True 185 """A ``bool`` which determines whether the spell is read only or 186 not. Default value is ``True``. Override this class attribute, and 187 set to ``False``, when subclassing a spell that must write files 188 back to the disk. 189 """ 190 191 SPELLNAME = None 192 """A ``str`` describing how to refer to the spell from the command line. 193 Override this class attribute when subclassing. 194 """ 195

196 - def __init__(self, toaster=None, data=None, stream=None):

197 """Initialize the spell data. 198 199 :param data: The file :attr:`data`. 200 :type data: :class:`~pyffi.object_models.FileFormat.Data` 201 :param stream: The file :attr:`stream`. 202 :type stream: ``file`` 203 :param toaster: The :attr:`toaster` this spell is called from (optional). 204 :type toaster: :class:`Toaster` 205 """ 206 self.data = data 207 self.stream = stream 208 self.toaster = toaster if toaster else Toaster()

209

210 - def _datainspect(self):

211 """This is called after :meth:`pyffi.object_models.FileFormat.Data.inspect` has 212 been called, and before :meth:`pyffi.object_models.FileFormat.Data.read` is 213 called. 214 215 :return: ``True`` if the file must be processed, ``False`` otherwise. 216 :rtype: ``bool`` 217 """ 218 # for the moment, this does nothing 219 return True

220

221 - def datainspect(self):

222 """This is called after :meth:`pyffi.object_models.FileFormat.Data.inspect` has 223 been called, and before :meth:`pyffi.object_models.FileFormat.Data.read` is 224 called. Override this function for customization. 225 226 :return: ``True`` if the file must be processed, ``False`` otherwise. 227 :rtype: ``bool`` 228 """ 229 # for nif: check if version applies, or 230 # check if spell block type is found 231 return True

232

233 - def _branchinspect(self, branch):

234 """Check if spell should be cast on this branch or not, based on 235 exclude and include options passed on the command line. You should 236 not need to override this function: if you need additional checks on 237 whether a branch must be parsed or not, override the :meth:`branchinspect` 238 method. 239 240 :param branch: The branch to check. 241 :type branch: :class:`~pyffi.utils.graph.GlobalNode` 242 :return: ``True`` if the branch must be processed, ``False`` otherwise. 243 :rtype: ``bool`` 244 """ 245 # fall back on the toaster implementation 246 return self.toaster.is_admissible_branch_class(branch.__class__)

247

248 - def branchinspect(self, branch):

249 """Like :meth:`_branchinspect`, but for customization: can be overridden to 250 perform an extra inspection (the default implementation always 251 returns ``True``). 252 253 :param branch: The branch to check. 254 :type branch: :class:`~pyffi.utils.graph.GlobalNode` 255 :return: ``True`` if the branch must be processed, ``False`` otherwise. 256 :rtype: ``bool`` 257 """ 258 return True

259

260 - def recurse(self, branch=None):

261 """Helper function which calls :meth:`_branchinspect` and :meth:`branchinspect` 262 on the branch, 263 if both successful then :meth:`branchentry` on the branch, and if this is 264 succesful it calls :meth:`recurse` on the branch's children, and 265 once all children are done, it calls :meth:`branchexit`. 266 267 Note that :meth:`_branchinspect` and :meth:`branchinspect` are not called upon 268 first entry of this function, that is, when called with :attr:`data` as 269 branch argument. Use :meth:`datainspect` to stop recursion into this branch. 270 271 Do not override this function. 272 273 :param branch: The branch to start the recursion from, or ``None`` 274 to recurse the whole tree. 275 :type branch: :class:`~pyffi.utils.graph.GlobalNode` 276 """ 277 # when called without arguments, recurse over the whole tree 278 if branch is None: 279 branch = self.data 280 # the root data element: datainspect has already been called 281 if branch is self.data: 282 self.toaster.msgblockbegin( 283 "--- %s ---" % self.SPELLNAME) 284 if self.dataentry(): 285 # spell returned True so recurse to children 286 # we use the abstract tree functions to parse the tree 287 # these are format independent! 288 for child in branch.get_global_child_nodes(): 289 self.recurse(child) 290 self.dataexit() 291 self.toaster.msgblockend() 292 elif self._branchinspect(branch) and self.branchinspect(branch): 293 self.toaster.msgblockbegin( 294 """~~~ %s [%s] ~~~""" 295 % (branch.__class__.__name__, 296 branch.get_global_display())) 297 # cast the spell on the branch 298 if self.branchentry(branch): 299 # spell returned True so recurse to children 300 # we use the abstract tree functions to parse the tree 301 # these are format independent! 302 for child in branch.get_global_child_nodes(): 303 self.recurse(child) 304 self.branchexit(branch) 305 self.toaster.msgblockend()

306

307 - def dataentry(self):

308 """Called before all blocks are recursed. 309 The default implementation simply returns ``True``. 310 You can access the data via :attr:`data`, and unlike in the 311 :meth:`datainspect` method, the full file has been processed at this stage. 312 313 Typically, you will override this function to perform a global 314 operation on the file data. 315 316 :return: ``True`` if the children must be processed, ``False`` otherwise. 317 :rtype: ``bool`` 318 """ 319 return True

320

321 - def branchentry(self, branch):

322 """Cast the spell on the given branch. First called with branch equal to 323 :attr:`data`'s children, then the grandchildren, and so on. 324 The default implementation simply returns ``True``. 325 326 Typically, you will override this function to perform an operation 327 on a particular block type and/or to stop recursion at particular 328 block types. 329 330 :param branch: The branch to cast the spell on. 331 :type branch: :class:`~pyffi.utils.graph.GlobalNode` 332 :return: ``True`` if the children must be processed, ``False`` otherwise. 333 :rtype: ``bool`` 334 """ 335 return True

336

337 - def branchexit(self, branch):

338 """Cast a spell on the given branch, after all its children, 339 grandchildren, have been processed, if :meth:`branchentry` returned 340 ``True`` on the given branch. 341 342 Typically, you will override this function to perform a particular 343 operation on a block type, but you rely on the fact that the children 344 must have been processed first. 345 346 :param branch: The branch to cast the spell on. 347 :type branch: :class:`~pyffi.utils.graph.GlobalNode` 348 """ 349 pass

350

351 - def dataexit(self):

352 """Called after all blocks have been processed, if :meth:`dataentry` 353 returned ``True``. 354 355 Typically, you will override this function to perform a final spell 356 operation, such as writing back the file in a special way, or making a 357 summary log. 358 """ 359 pass

360 361 @classmethod

362 - def toastentry(cls, toaster):

363 """Called just before the toaster starts processing 364 all files. If it returns ``False``, then the spell is not used. 365 The default implementation simply returns ``True``. 366 367 For example, if the spell only acts on a particular block type, but 368 that block type is excluded, then you can use this function to flag 369 that this spell can be skipped. You can also use this function to 370 initialize statistics data to be aggregated from files, to 371 initialize a log file, and so. 372 373 :param toaster: The toaster this spell is called from. 374 :type toaster: :class:`Toaster` 375 :return: ``True`` if the spell applies, ``False`` otherwise. 376 :rtype: ``bool`` 377 """ 378 return True

379 380 @classmethod

381 - def toastexit(cls, toaster):

382 """Called when the toaster has finished processing 383 all files. 384 385 :param toaster: The toaster this spell is called from. 386 :type toaster: :class:`Toaster` 387 """ 388 pass

389 390 @classmethod

391 - def get_toast_stream(cls, toaster, filename, test_exists=False):

392 """Returns the stream that the toaster will write to. The 393 default implementation calls ``toaster.get_toast_stream``, but 394 spells that write to different file(s) can override this 395 method. 396 """ 397 return toaster.get_toast_stream(filename, test_exists=test_exists)

398

399 -class SpellGroupBase(Spell):

400 """Base class for grouping spells. This implements all the spell grouping 401 functions that fall outside of the actual recursing (:meth:`__init__`, 402 :meth:`toastentry`, :meth:`_datainspect`, :meth:`datainspect`, and :meth:`toastexit`). 403 """ 404 405 SPELLCLASSES = [] 406 """List of :class:`Spell`\ s of this group (not instantiated).""" 407 408 ACTIVESPELLCLASSES = [] 409 """List of active spells of this group (not instantiated). 410 This list is automatically built when :meth:`toastentry` is called. 411 """ 412 413 spells = [] 414 """List of active spell instances.""" 415

416 - def __init__(self, toaster=None, data=None, stream=None):

417 """Initialize the spell data for all given spells. 418 419 :param toaster: The toaster this spell is called from. 420 :type toaster: :class:`Toaster` 421 :param data: The file data. 422 :type data: :class:`pyffi.object_models.FileFormat.Data` 423 :param stream: The file stream. 424 :type stream: ``file`` 425 """ 426 # call base class constructor 427 Spell.__init__(self, toaster=toaster, data=data, stream=stream) 428 # set up the list of spells 429 self.spells = [spellclass(toaster=toaster, data=data, stream=stream) 430 for spellclass in self.ACTIVESPELLCLASSES]

431

432 - def datainspect(self):

433 """Inspect every spell with L{Spell.datainspect} and keep 434 those spells that must be cast.""" 435 self.spells = [spell for spell in self.spells 436 if spell.datainspect()] 437 return bool(self.spells)

438 439 @classmethod

440 - def toastentry(cls, toaster):

441 cls.ACTIVESPELLCLASSES = [ 442 spellclass for spellclass in cls.SPELLCLASSES 443 if spellclass.toastentry(toaster)] 444 return bool(cls.ACTIVESPELLCLASSES)

445 446 @classmethod

447 - def toastexit(cls, toaster):

448 for spellclass in cls.ACTIVESPELLCLASSES: 449 spellclass.toastexit(toaster)

450

451 -class SpellGroupSeriesBase(SpellGroupBase):

452 """Base class for running spells in series."""

453 - def recurse(self, branch=None):

454 """Recurse spells in series.""" 455 for spell in self.spells: 456 spell.recurse(branch)

457 458 # the following functions must NEVER be called in series of spells 459 # everything is handled by the recurse function 460

461 - def branchinspect(self, branch):

462 raise RuntimeError("use recurse")

463

464 - def branchentry(self, branch):

465 raise RuntimeError("use recurse")

466

467 - def dataexit(self):

468 raise RuntimeError("use recurse")

469

470 - def dataentry(self):

471 raise RuntimeError("use recurse")

472

473 - def dataexit(self):

474 raise RuntimeError("use recurse")

475 476 @property

477 - def changed(self):

478 return any(spell.changed for spell in self.spells)

479

480 -class SpellGroupParallelBase(SpellGroupBase):

481 """Base class for running spells in parallel (that is, with only 482 a single recursion in the tree). 483 """

484 - def branchinspect(self, branch):

485 """Inspect spells with :meth:`Spell.branchinspect` (not all checks are 486 executed, only keeps going until a spell inspection returns ``True``). 487 """ 488 return any(spell.branchinspect(branch) for spell in self.spells)

489

490 - def branchentry(self, branch):

491 """Run all spells.""" 492 # not using any: we want all entry code to be executed 493 return bool([spell.branchentry(branch) for spell in self.spells])

494

495 - def branchexit(self, branch):

496 for spell in self.spells: 497 spell.branchexit(branch)

498

499 - def dataentry(self):

500 """Look into every spell with :meth:`Spell.dataentry`.""" 501 self.spells = [spell for spell in self.spells 502 if spell.dataentry()] 503 return bool(self.spells)

504

505 - def dataexit(self):

506 """Look into every spell with :meth:`Spell.dataexit`.""" 507 for spell in self.spells: 508 spell.dataexit()

509 510 @property

511 - def changed(self):

512 return any(spell.changed for spell in self.spells)

513

514 -def SpellGroupSeries(*args):

515 """Class factory for grouping spells in series.""" 516 return type("".join(spellclass.__name__ for spellclass in args), 517 (SpellGroupSeriesBase,), 518 {"SPELLCLASSES": args, 519 "SPELLNAME": 520 " | ".join(spellclass.SPELLNAME for spellclass in args), 521 "READONLY": 522 all(spellclass.READONLY for spellclass in args)})

523

524 -def SpellGroupParallel(*args):

525 """Class factory for grouping spells in parallel.""" 526 return type("".join(spellclass.__name__ for spellclass in args), 527 (SpellGroupParallelBase,), 528 {"SPELLCLASSES": args, 529 "SPELLNAME": 530 " & ".join(spellclass.SPELLNAME for spellclass in args), 531 "READONLY": 532 all(spellclass.READONLY for spellclass in args)})

533

534 -class SpellApplyPatch(Spell):

535 """A spell for applying a patch on files.""" 536 537 SPELLNAME = "applypatch" 538

539 - def datainspect(self):

540 """There is no need to read the whole file, so we apply the patch 541 already at inspection stage, and stop the spell process by returning 542 ``False``. 543 544 :return: ``False`` 545 :rtype: ``bool`` 546 """ 547 # get the patch command (if there is one) 548 patchcmd = self.toaster.options["patchcmd"] 549 if not patchcmd: 550 raise ValueError("must specify a patch command") 551 # first argument is always the stream, by convention 552 oldfile = self.stream 553 oldfilename = oldfile.name 554 newfilename = oldfilename + ".patched" 555 patchfilename = oldfilename + ".patch" 556 self.toaster.msg("writing %s..." % newfilename) 557 # close all files before calling external command 558 oldfile.close() 559 subprocess.call([patchcmd, oldfilename, newfilename, patchfilename]) 560 561 # do not go further, spell is done 562 return False

563

564 565 -class fake_logger:

566 """Simple logger for testing.""" 567 level = logging.DEBUG 568 569 @classmethod

570 - def _log(cls, level, level_str, msg):

571 # do not actually log, just print 572 if level >= cls.level: 573 print("pyffi.toaster:%s:%s" % (level_str, msg))

574 575 @classmethod

576 - def error(cls, msg):

577 cls._log(logging.ERROR, "ERROR", msg)

578 579 @classmethod

580 - def warn(cls, msg):

581 cls._log(logging.WARNING, "WARNING", msg)

582 583 @classmethod

584 - def info(cls, msg):

585 cls._log(logging.INFO, "INFO", msg)

586 587 @classmethod

588 - def debug(cls, msg):

589 cls._log(logging.DEBUG, "DEBUG", msg)

590 591 @classmethod

592 - def setLevel(cls, level):

593 cls.level = level

594

595 -def _toaster_job(args):

596 """For multiprocessing. This function creates a new toaster, with the 597 given options and spells, and calls the toaster on filename. 598 """ 599 600 class multiprocessing_fake_logger(fake_logger): 601 """Simple logger which works well along with multiprocessing 602 on all platforms. 603 """ 604 @classmethod 605 def _log(cls, level, level_str, msg): 606 # do not actually log, just print 607 if level >= cls.level: 608 print("pyffi.toaster:%i:%s:%s" 609 % (multiprocessing.current_process().pid, 610 level_str, msg))

611 612 toasterclass, filename, options, spellnames = args 613 toaster = toasterclass(options=options, spellnames=spellnames, 614 logger=multiprocessing_fake_logger) 615 616 # toast entry code 617 if not toaster.spellclass.toastentry(toaster): 618 self.msg("spell does not apply! quiting early...") 619 return 620 621 # toast single file 622 stream = open(filename, mode='rb' if toaster.spellclass.READONLY else 'r+b') 623 toaster._toast(stream) 624 625 # toast exit code 626 toaster.spellclass.toastexit(toaster) 627 628 # CPU_COUNT is used for default number of jobs 629 if multiprocessing: 630 try: 631 CPU_COUNT = multiprocessing.cpu_count() 632 except NotImplementedError: 633 CPU_COUNT = 1 634 else: 635 CPU_COUNT = 1

636 637 -class Toaster(object):

638 """Toaster base class. Toasters run spells on large quantities of files. 639 They load each file and pass the data structure to any number of spells. 640 """ 641 642 FILEFORMAT = pyffi.object_models.FileFormat 643 """The file format class (a subclass of 644 :class:`~pyffi.object_models.FileFormat`).""" 645 646 SPELLS = [] 647 """List of all available :class:`~pyffi.spells.Spell` classes.""" 648 649 EXAMPLES = "" 650 """Some examples which describe typical use of the toaster.""" 651 652 ALIASDICT = {} 653 """Dictionary with aliases for spells.""" 654 655 DEFAULT_OPTIONS = dict( 656 raisetesterror=False, verbose=1, pause=False, 657 exclude=[], include=[], examples=False, 658 spells=False, 659 interactive=True, 660 helpspell=False, dryrun=False, prefix="", suffix="", arg="", 661 createpatch=False, applypatch=False, diffcmd="", patchcmd="", 662 series=False, 663 skip=[], only=[], 664 jobs=CPU_COUNT, refresh=32, 665 sourcedir="", destdir="", 666 archives=False, 667 resume=False, 668 inifile="") 669 670 """List of spell classes of the particular :class:`Toaster` instance.""" 671 672 options = {} 673 """The options of the toaster, as ``dict``.""" 674 675 spellnames = [] 676 """A list of the names of the spells.""" 677 678 top = "" 679 """Name of the top folder to toast.""" 680 681 indent = 0 682 """An ``int`` which describes the current indentation level for messages.""" 683 684 logger = logging.getLogger("pyffi.toaster") 685 """A :class:`logging.Logger` for toaster log messages.""" 686 687 include_types = [] 688 """Tuple of types corresponding to the include key of :attr:`options`.""" 689 690 exclude_types = [] 691 """Tuple of types corresponding to the exclude key of :attr:`options`.""" 692 693 only_regexs = [] 694 """Tuple of regular expressions corresponding to the only key of 695 :attr:`options`.""" 696 697 skip_regexs = [] 698 """Tuple of regular expressions corresponding to the skip key of 699 :attr:`options`.""" 700

701 - def __init__(self, spellclass=None, options=None, spellnames=None, 702 logger=None):

703 """Initialize the toaster. 704 705 :param spellclass: Deprecated, use spellnames. 706 :type spellclass: :class:`Spell` 707 :param options: The options (as keyword arguments). 708 :type options: ``dict`` 709 :param spellnames: List of names of spells. 710 :type spellnames: ``list`` of ``str`` 711 """ 712 self.options = deepcopy(self.DEFAULT_OPTIONS) 713 self.spellnames = spellnames if spellnames else [] 714 if logger: 715 # override default logger 716 self.logger = logger 717 if options: 718 self.options.update(options) 719 self.indent = 0 720 # update options and spell class 721 self._update_options() 722 if spellnames: 723 self._update_spellclass() 724 else: 725 # deprecated 726 self.spellclass = spellclass

727

728 - def _update_options(self):

729 """Synchronize some fields with given options.""" 730 # set verbosity level (also of self.logger, in case of a custom one) 731 if self.options["verbose"] <= 0: 732 logging.getLogger("pyffi").setLevel(logging.WARNING) 733 self.logger.setLevel(logging.WARNING) 734 elif self.options["verbose"] == 1: 735 logging.getLogger("pyffi").setLevel(logging.INFO) 736 self.logger.setLevel(logging.INFO) 737 else: 738 logging.getLogger("pyffi").setLevel(logging.DEBUG) 739 self.logger.setLevel(logging.DEBUG) 740 # check errors 741 if self.options["createpatch"] and self.options["applypatch"]: 742 raise ValueError( 743 "options --diff and --patch are mutually exclusive") 744 if self.options["diffcmd"] and not(self.options["createpatch"]): 745 raise ValueError( 746 "option --diff-cmd can only be used with --diff") 747 if self.options["patchcmd"] and not(self.options["applypatch"]): 748 raise ValueError( 749 "option --patch-cmd can only be used with --patch") 750 # multiprocessing available? 751 if (multiprocessing is None) and self.options["jobs"] > 1: 752 self.logger.warn( 753 "multiprocessing not supported on this platform") 754 self.options["jobs"] = 1 755 # update include and exclude types 756 self.include_types = tuple( 757 getattr(self.FILEFORMAT, block_type) 758 for block_type in self.options["include"]) 759 self.exclude_types = tuple( 760 getattr(self.FILEFORMAT, block_type) 761 for block_type in self.options["exclude"]) 762 # update skip and only regular expressions 763 self.skip_regexs = tuple( 764 re.compile(regex) for regex in self.options["skip"]) 765 self.only_regexs = tuple( 766 re.compile(regex) for regex in self.options["only"])

767

768 - def _update_spellclass(self):

769 """Update spell class from given list of spell names.""" 770 # get spell classes 771 spellclasses = [] 772 if not self.spellnames: 773 raise ValueError("no spells specified") 774 for spellname in self.spellnames: 775 # convert old names 776 if spellname in self.ALIASDICT: 777 self.logger.warning( 778 "The %s spell is deprecated and will be removed" 779 " from a future release; use the %s spell as a" 780 " replacement" % (spellname, self.ALIASDICT[spellname])) 781 spellname = self.ALIASDICT[spellname] 782 # find the spell 783 spellklasses = [spellclass for spellclass in self.SPELLS 784 if spellclass.SPELLNAME == spellname] 785 if not spellklasses: 786 raise ValueError( 787 "%s is not a known spell" % spellname) 788 if len(spellklasses) > 1: 789 raise ValueError( 790 "multiple spells are called %s (BUG?)" % spellname) 791 spellclasses.extend(spellklasses) 792 # create group of spells 793 if len(spellclasses) > 1: 794 if self.options["series"]: 795 self.spellclass = SpellGroupSeries(*spellclasses) 796 else: 797 self.spellclass = SpellGroupParallel(*spellclasses) 798 else: 799 self.spellclass = spellclasses[0]

800

801 - def msg(self, message):

802 """Write log message with :meth:`logger.info`, taking into account 803 :attr:`indent`. 804 805 :param message: The message to write. 806 :type message: ``str`` 807 """ 808 for line in message.split("\n"): 809 self.logger.info(" " * self.indent + line)

810

811 - def msgblockbegin(self, message):

812 """Acts like :meth:`msg`, but also increases :attr:`indent` after writing the 813 message.""" 814 self.msg(message) 815 self.indent += 1

816

817 - def msgblockend(self, message=None):

818 """Acts like :meth:`msg`, but also decreases :attr:`indent` before writing the 819 message, but if the message argument is ``None``, then no message is 820 printed.""" 821 self.indent -= 1 822 if not(message is None): 823 self.msg(message)

824

825 - def is_admissible_branch_class(self, branchtype):

826 """Helper function which checks whether a given branch type should 827 have spells cast on it or not, based in exclude and include options. 828 829 >>> from pyffi.formats.nif import NifFormat 830 >>> class MyToaster(Toaster): 831 ... FILEFORMAT = NifFormat 832 >>> toaster = MyToaster() # no include or exclude: all admissible 833 >>> toaster.is_admissible_branch_class(NifFormat.NiProperty) 834 True 835 >>> toaster.is_admissible_branch_class(NifFormat.NiNode) 836 True 837 >>> toaster.is_admissible_branch_class(NifFormat.NiAVObject) 838 True 839 >>> toaster.is_admissible_branch_class(NifFormat.NiLODNode) 840 True 841 >>> toaster.is_admissible_branch_class(NifFormat.NiMaterialProperty) 842 True 843 >>> toaster = MyToaster(options={"exclude": ["NiProperty", "NiNode"]}) 844 >>> toaster.is_admissible_branch_class(NifFormat.NiProperty) 845 False 846 >>> toaster.is_admissible_branch_class(NifFormat.NiNode) 847 False 848 >>> toaster.is_admissible_branch_class(NifFormat.NiAVObject) 849 True 850 >>> toaster.is_admissible_branch_class(NifFormat.NiLODNode) 851 False 852 >>> toaster.is_admissible_branch_class(NifFormat.NiMaterialProperty) 853 False 854 >>> toaster = MyToaster(options={"include": ["NiProperty", "NiNode"]}) 855 >>> toaster.is_admissible_branch_class(NifFormat.NiProperty) 856 True 857 >>> toaster.is_admissible_branch_class(NifFormat.NiNode) 858 True 859 >>> toaster.is_admissible_branch_class(NifFormat.NiAVObject) 860 False 861 >>> toaster.is_admissible_branch_class(NifFormat.NiLODNode) # NiNodes are! 862 True 863 >>> toaster.is_admissible_branch_class(NifFormat.NiMaterialProperty) # NiProperties are! 864 True 865 >>> toaster = MyToaster(options={"include": ["NiProperty", "NiNode"], "exclude": ["NiMaterialProperty", "NiLODNode"]}) 866 >>> toaster.is_admissible_branch_class(NifFormat.NiProperty) 867 True 868 >>> toaster.is_admissible_branch_class(NifFormat.NiNode) 869 True 870 >>> toaster.is_admissible_branch_class(NifFormat.NiAVObject) 871 False 872 >>> toaster.is_admissible_branch_class(NifFormat.NiLODNode) 873 False 874 >>> toaster.is_admissible_branch_class(NifFormat.NiSwitchNode) 875 True 876 >>> toaster.is_admissible_branch_class(NifFormat.NiMaterialProperty) 877 False 878 >>> toaster.is_admissible_branch_class(NifFormat.NiAlphaProperty) 879 True 880 """ 881 #print("checking %s" % branchtype.__name__) # debug 882 # check that block is not in exclude... 883 if not issubclass(branchtype, self.exclude_types): 884 # not excluded! 885 # check if it is included 886 if not self.include_types: 887 # if no include list is given, then assume included by default 888 # so, the block is admissible 889 return True 890 elif issubclass(branchtype, self.include_types): 891 # included as well! the block is admissible 892 return True 893 # not admissible 894 #print("not admissible") # debug 895 return False

896 897 @staticmethod

898 - def parse_inifile(option, opt, value, parser, toaster=None):

899 r"""Initializes spell classes and options from an ini file. 900 901 >>> import pyffi.spells.nif 902 >>> import pyffi.spells.nif.modify 903 >>> class NifToaster(pyffi.spells.nif.NifToaster): 904 ... SPELLS = [pyffi.spells.nif.modify.SpellDelBranches] 905 >>> import tempfile 906 >>> cfg = tempfile.NamedTemporaryFile(delete=False) 907 >>> cfg.write("[main]\n") 908 >>> cfg.write("spell = modify_delbranches\n") 909 >>> cfg.write("folder = tests/nif/test_vertexcolor.nif\n") 910 >>> cfg.write("[options]\n") 911 >>> cfg.write("source-dir = tests/\n") 912 >>> cfg.write("dest-dir = _tests/\n") 913 >>> cfg.write("exclude = NiVertexColorProperty NiStencilProperty\n") 914 >>> cfg.write("skip = 'testing quoted string' normal_string\n") 915 >>> cfg.close() 916 >>> toaster = NifToaster(logger=fake_logger) 917 >>> import sys 918 >>> sys.argv = [ 919 ... "niftoaster.py", 920 ... "--ini-file=utilities/toaster/default.ini", 921 ... "--ini-file=%s" % cfg.name, 922 ... "--noninteractive", "--jobs=1"] 923 >>> toaster.cli() 924 pyffi.toaster:INFO:=== tests/nif/test_vertexcolor.nif === 925 pyffi.toaster:INFO: --- modify_delbranches --- 926 pyffi.toaster:INFO: ~~~ NiNode [Scene Root] ~~~ 927 pyffi.toaster:INFO: ~~~ NiTriStrips [Cube] ~~~ 928 pyffi.toaster:INFO: ~~~ NiStencilProperty [] ~~~ 929 pyffi.toaster:INFO: stripping this branch 930 pyffi.toaster:INFO: ~~~ NiSpecularProperty [] ~~~ 931 pyffi.toaster:INFO: ~~~ NiMaterialProperty [Material] ~~~ 932 pyffi.toaster:INFO: ~~~ NiVertexColorProperty [] ~~~ 933 pyffi.toaster:INFO: stripping this branch 934 pyffi.toaster:INFO: ~~~ NiTriStripsData [] ~~~ 935 pyffi.toaster:INFO:creating destination path _tests/nif 936 pyffi.toaster:INFO: writing _tests/nif/test_vertexcolor.nif 937 pyffi.toaster:INFO:Finished. 938 >>> import os 939 >>> os.remove(cfg.name) 940 >>> os.remove("_tests/nif/test_vertexcolor.nif") 941 >>> os.rmdir("_tests/nif/") 942 >>> os.rmdir("_tests/") 943 >>> for name, value in sorted(toaster.options.items()): 944 ... print("%s: %s" % (name, value)) 945 applypatch: False 946 archives: False 947 arg: 948 createpatch: False 949 destdir: _tests/ 950 diffcmd: 951 dryrun: False 952 examples: False 953 exclude: ['NiVertexColorProperty', 'NiStencilProperty'] 954 helpspell: False 955 include: [] 956 inifile: 957 interactive: False 958 jobs: 1 959 only: [] 960 patchcmd: 961 pause: True 962 prefix: 963 raisetesterror: False 964 refresh: 32 965 resume: True 966 series: False 967 skip: ['testing quoted string', 'normal_string'] 968 sourcedir: tests/ 969 spells: False 970 suffix: 971 verbose: 1 972 """ 973 ini_parser = ConfigParser() 974 # read config file(s) 975 ini_parser.read(value) 976 # process all options 977 if ini_parser.has_section("options"): 978 for opt_str, opt_values in ini_parser.items("options"): 979 option = parser._long_opt["--" + opt_str] 980 for opt_value in shlex.split(opt_values): 981 option.process(opt_str, opt_value, parser.values, parser) 982 # get spells and top folder 983 if ini_parser.has_section("main"): 984 if ini_parser.has_option("main", "spell"): 985 toaster.spellnames.extend(ini_parser.get("main", "spell").split()) 986 if ini_parser.has_option("main", "folder"): 987 toaster.top = ini_parser.get("main", "folder")

988

989 - def cli(self):

990 """Command line interface: initializes spell classes and options from 991 the command line, and run the :meth:`toast` method. 992 """ 993 # parse options and positional arguments 994 usage = "%prog [options] <spell1> <spell2> ... <file>|<folder>" 995 description = ( 996 "Apply the spells <spell1>, <spell2>, and so on," 997 " on <file>, or recursively on <folder>.") 998 errormessage_numargs = ( 999 "incorrect number of arguments (use the --help option for help)") 1000 1001 parser = optparse.OptionParser( 1002 usage, 1003 version="%%prog (PyFFI %s)" % pyffi.__version__, 1004 description=description) 1005 parser.add_option( 1006 "--archives", dest="archives", 1007 action="store_true", 1008 help="also parse files inside archives") 1009 parser.add_option( 1010 "-a", "--arg", dest="arg", 1011 type="string", 1012 metavar="ARG", 1013 help="pass argument ARG to each spell") 1014 parser.add_option( 1015 "--dest-dir", dest="destdir", 1016 type="string", 1017 metavar="DESTDIR", 1018 help= 1019 "write files to DESTDIR" 1020 " instead of overwriting the original;" 1021 " this is done by replacing SOURCEDIR by DESTDIR" 1022 " in all source file paths") 1023 parser.add_option( 1024 "--diff", dest="createpatch", 1025 action="store_true", 1026 help= 1027 "write a binary patch" 1028 " instead of overwriting the original") 1029 parser.add_option( 1030 "--diff-cmd", dest="diffcmd", 1031 type="string", 1032 metavar="CMD", 1033 help= 1034 "use CMD as diff command; this command must accept precisely" 1035 " 3 arguments: 'CMD oldfile newfile patchfile'.") 1036 parser.add_option( 1037 "--dry-run", dest="dryrun", 1038 action="store_true", 1039 help= 1040 "save modification to temporary file" 1041 " instead of overwriting the original" 1042 " (for debugging)") 1043 parser.add_option( 1044 "--examples", dest="examples", 1045 action="store_true", 1046 help="show examples of usage and exit") 1047 parser.add_option( 1048 "--help-spell", dest="helpspell", 1049 action="store_true", 1050 help="show help specific to the given spells") 1051 parser.add_option( 1052 "-i", "--include", dest="include", 1053 type="string", 1054 action="append", 1055 metavar="BLOCK", 1056 help= 1057 "include only block type BLOCK in spell; if this option is" 1058 " not specified, then all block types are included except" 1059 " those specified under --exclude; include multiple block" 1060 " types by specifying this option more than once") 1061 parser.add_option( 1062 "--ini-file", dest="inifile", 1063 type="string", 1064 action="callback", 1065 callback=self.parse_inifile, 1066 callback_kwargs={'toaster': self}, 1067 metavar="FILE", 1068 help= 1069 "read all options from FILE; if specified, all other arguments" 1070 " are ignored; to take options from multiple ini files, specify" 1071 " more than once") 1072 parser.add_option( 1073 "-j", "--jobs", dest="jobs", 1074 type="int", 1075 metavar="JOBS", 1076 help="allow JOBS jobs at once [default: %default]") 1077 parser.add_option( 1078 "--noninteractive", dest="interactive", 1079 action="store_false", 1080 help="non-interactive session (overwrites files without warning)") 1081 parser.add_option( 1082 "--only", dest="only", 1083 type="string", 1084 action="append", 1085 metavar="REGEX", 1086 help= 1087 "only toast files whose names" 1088 " (i) contain the regular expression REGEX, and" 1089 " (ii) do not contain any regular expression specified with --skip;" 1090 " if specified multiple times, the expressions are 'ored'") 1091 parser.add_option( 1092 "--overwrite", dest="resume", 1093 action="store_false", 1094 help="overwrite existing files (also see --resume)") 1095 parser.add_option( 1096 "--patch", dest="applypatch", 1097 action="store_true", 1098 help="apply all binary patches") 1099 parser.add_option( 1100 "--patch-cmd", dest="patchcmd", 1101 type="string", 1102 metavar="CMD", 1103 help= 1104 "use CMD as patch command; this command must accept precisely " 1105 "3 arguments: 'CMD oldfile newfile patchfile'.""") 1106 parser.add_option( 1107 "-p", "--pause", dest="pause", 1108 action="store_true", 1109 help="pause when done") 1110 parser.add_option( 1111 "--prefix", dest="prefix", 1112 type="string", 1113 metavar="PREFIX", 1114 help= 1115 "prepend PREFIX to file name when saving modification" 1116 " instead of overwriting the original") 1117 parser.add_option( 1118 "-r", "--raise", dest="raisetesterror", 1119 action="store_true", 1120 help="raise exception on errors during the spell (for debugging)") 1121 parser.add_option( 1122 "--refresh", dest="refresh", 1123 type="int", 1124 metavar="REFRESH", 1125 help= 1126 "start new process pool every JOBS * REFRESH files" 1127 " if JOBS is 2 or more" 1128 " (when processing a large number of files, this prevents" 1129 " leaking memory on some operating systems) [default: %default]") 1130 parser.add_option( 1131 "--resume", dest="resume", 1132 action="store_true", 1133 help="do not overwrite existing files") 1134 parser.add_option( 1135 "--series", dest="series", 1136 action="store_true", 1137 help="run spells in series rather than in parallel") 1138 parser.add_option( 1139 "--skip", dest="skip", 1140 type="string", 1141 action="append", 1142 metavar="REGEX", 1143 help= 1144 "skip all files whose names contain the regular expression REGEX" 1145 " (takes precedence over --only);" 1146 " if specified multiple times, the expressions are 'ored'") 1147 parser.add_option( 1148 "--source-dir", dest="sourcedir", 1149 type="string", 1150 metavar="SOURCEDIR", 1151 help= 1152 "see --dest-dir") 1153 parser.add_option( 1154 "--spells", dest="spells", 1155 action="store_true", 1156 help="list all spells and exit") 1157 parser.add_option( 1158 "--suffix", dest="suffix", 1159 type="string", 1160 metavar="SUFFIX", 1161 help="append SUFFIX to file name when saving modification" 1162 " instead of overwriting the original") 1163 parser.add_option( 1164 "-v", "--verbose", dest="verbose", 1165 type="int", 1166 metavar="LEVEL", 1167 help="verbosity level: 0, 1, or 2 [default: %default]") 1168 parser.add_option( 1169 "-x", "--exclude", dest="exclude", 1170 type="string", 1171 action="append", 1172 metavar="BLOCK", 1173 help= 1174 "exclude block type BLOCK from spell; exclude multiple" 1175 " block types by specifying this option more than once") 1176 parser.set_defaults(**deepcopy(self.DEFAULT_OPTIONS)) 1177 (options, args) = parser.parse_args() 1178 1179 # convert options to dictionary 1180 self.options = {} 1181 for optionname in dir(options): 1182 # skip default attributes of optparse.Values 1183 if not optionname in dir(optparse.Values): 1184 self.options[optionname] = getattr(options, optionname) 1185 1186 # update options 1187 self._update_options() 1188 1189 # check if we had examples and/or spells: quit 1190 if options.spells: 1191 for spellclass in self.SPELLS: 1192 print(spellclass.SPELLNAME) 1193 return 1194 if options.examples: 1195 print(self.EXAMPLES) 1196 return 1197 1198 # check if we are applying patches 1199 if options.applypatch: 1200 if len(args) > 1: 1201 parser.error("when using --patch, do not specify a spell") 1202 # set spell class to applying patch 1203 self.spellclass = SpellApplyPatch 1204 # set top 1205 if args: 1206 self.top = args[-1] 1207 elif not self.top: 1208 parser.error("no folder or file specified") 1209 else: 1210 # get spell names and top 1211 if options.helpspell: 1212 # special case: --spell-help would not have a top specified 1213 self.spellnames = args[:] 1214 self._update_spellclass() 1215 self.msg(self.spellclass.__doc__) 1216 return 1217 if not args: 1218 # no args: error if no top or no spells 1219 if not(self.top and self.spellnames): 1220 parser.error(errormessage_numargs) 1221 elif len(args) == 1: 1222 # single argument is top, error if no spells 1223 if not(self.spellnames): 1224 parser.error(errormessage_numargs) 1225 self.top = args[-1] 1226 else: 1227 # all arguments, except the last one, are spells 1228 self.spellnames.extend(args[:-1]) 1229 # last argument is top 1230 self.top = args[-1] 1231 # update the spell class 1232 self._update_spellclass() 1233 1234 if not self.options["archives"]: 1235 self.toast(self.top) 1236 else: 1237 self.toast_archives(self.top) 1238 1239 # signal the end 1240 self.logger.info("Finished.") 1241 if options.pause and options.interactive: 1242 raw_input("Press enter...")

1243

1244 - def inspect_filename(self, filename):

1245 """Returns whether to toast a filename or not, based on 1246 skip_regexs and only_regexs. 1247 """ 1248 if any(regex.search(filename) for regex in self.skip_regexs): 1249 # found some --skip regex, so do not toast 1250 return False 1251 if not self.only_regexs: 1252 # --only not specified: then by default take all files 1253 return True 1254 if any(regex.search(filename) for regex in self.only_regexs): 1255 # found at least one --only regex, so toast 1256 return True 1257 else: 1258 # no --only found, so do not toast 1259 return False

1260

1261 - def toast(self, top):

1262 """Walk over all files in a directory tree and cast spells 1263 on every file. 1264 1265 :param top: The directory or file to toast. 1266 :type top: str 1267 """ 1268 1269 def file_pools(chunksize): 1270 """Helper function which generates list of files, sorted by size, 1271 in chunks of given size. 1272 """ 1273 all_files = pyffi.utils.walk( 1274 top, onerror=None, 1275 re_filename=self.FILEFORMAT.RE_FILENAME) 1276 while True: 1277 # fetch chunksize files from all files 1278 file_pool = [ 1279 filename for i, filename in izip( 1280 xrange(chunksize), all_files)] 1281 if not file_pool: 1282 # done! 1283 break 1284 # sort files by size 1285 file_pool.sort(key=os.path.getsize, reverse=True) 1286 yield file_pool

1287 1288 # toast entry code 1289 if not self.spellclass.toastentry(self): 1290 self.msg("spell does not apply! quiting early...") 1291 return 1292 1293 # some defaults are different from the defaults defined in 1294 # the cli function: these defaults are reasonable for when the 1295 # toaster is called NOT from the command line 1296 # whereas the cli function defines defaults that are reasonable 1297 # when the toaster is called from the command line 1298 # in particular, when calling from the command line, the script 1299 # is much more verbose by default 1300 1301 pause = self.options.get("pause", False) 1302 1303 # do not ask for confirmation (!= cli default) 1304 interactive = self.options.get("interactive", False) 1305 1306 dryrun = self.options.get("dryrun", False) 1307 prefix = self.options.get("prefix", "") 1308 suffix = self.options.get("suffix", "") 1309 destdir = self.options.get("destdir", "") 1310 sourcedir = self.options.get("sourcedir", "") 1311 createpatch = self.options.get("createpatch", False) 1312 applypatch = self.options.get("applypatch", False) 1313 jobs = self.options.get("jobs", CPU_COUNT) 1314 1315 # get source directory if not specified 1316 if not sourcedir: 1317 # set default source directory 1318 if os.path.isfile(top): 1319 sourcedir = os.path.dirname(top) 1320 else: 1321 sourcedir = top 1322 # store the option (so spells can use it) 1323 self.options["sourcedir"] = sourcedir 1324 1325 # check that top starts with sourcedir 1326 if not top.startswith(sourcedir): 1327 raise ValueError( 1328 "invalid --source-dir: %s does not start with %s" 1329 % (top, sourcedir)) 1330 1331 # warning 1332 if ((not self.spellclass.READONLY) and (not dryrun) 1333 and (not prefix) and (not createpatch) 1334 and interactive and (not suffix) and (not destdir)): 1335 print("""\ 1336 This script will modify your files, in particular if something goes wrong it 1337 may destroy them. Make a backup of your files before running this script. 1338 """) 1339 if not raw_input( 1340 "Are you sure that you want to proceed? [n/y] ") in ("y", "Y"): 1341 self.logger.info("Script aborted by user.") 1342 if pause: 1343 raw_input("Press enter...") 1344 return 1345 1346 # walk over all streams, and create a data instance for each of them 1347 # inspect the file but do not yet read in full 1348 if jobs == 1: 1349 for stream in self.FILEFORMAT.walk( 1350 top, mode='rb' if self.spellclass.READONLY else 'r+b'): 1351 pass # to set a breakpoint 1352 self._toast(stream) 1353 # force free memory (helps when parsing many very large files) 1354 gc.collect() 1355 pass # to set a breakpoint 1356 else: 1357 chunksize = self.options["refresh"] * self.options["jobs"] 1358 self.msg("toasting with %i threads in chunks of %i files" 1359 % (jobs, chunksize)) 1360 for file_pool in file_pools(chunksize): 1361 self.logger.debug("process file pool:") 1362 for filename in file_pool: 1363 self.logger.debug(" " + filename) 1364 pool = multiprocessing.Pool(processes=jobs) 1365 # force chunksize=1 for the pool 1366 # this makes sure that the largest files (which come first 1367 # in the pool) are processed in parallel 1368 result = pool.map_async( 1369 _toaster_job, 1370 ((self.__class__, filename, self.options, self.spellnames) 1371 for filename in file_pool), 1372 chunksize=1) 1373 # specify timeout, so CTRL-C works 1374 # 99999999 is about 3 years, should be long enough... :-) 1375 result.wait(timeout=99999999) 1376 pool.close() 1377 pool.join() 1378 1379 # toast exit code 1380 self.spellclass.toastexit(self)

1381

1382 - def toast_archives(self, top):

1383 """Toast all files in all archives.""" 1384 if not self.FILEFORMAT.ARCHIVE_CLASSES: 1385 self.logger.info("No known archives contain this file format.") 1386 # walk over all files, and pick archives as we go 1387 for filename_in in pyffi.utils.walk(top): 1388 for ARCHIVE_CLASS in self.FILEFORMAT.ARCHIVE_CLASSES: 1389 # check if extension matches 1390 if not ARCHIVE_CLASS.RE_FILENAME.match(filename_in): 1391 continue 1392 # open the archive 1393 try: 1394 archive_in = ARCHIVE_CLASS.Data(name=filename_in, mode='r') 1395 except ValueError: 1396 self.logger.warn("archive format not recognized, skipped") 1397 continue 1398 # toast all members in the archive 1399 # and save them to a temporary archive as we go 1400 if not self.spellclass.READONLY: 1401 for member in archive_in.get_members(): 1402 self._toast_member(member) 1403 else: 1404 file_out = tempfile.TemporaryFile() 1405 archive_out = ARCHIVE_CLASS.Data(fileobj=file_out, mode='w') 1406 for member in archive_in.get_members(): 1407 self._toast(member) 1408 archive_out.add(member) 1409 archive_out.close() 1410 archive_in.close()

1411

1412 - def _toast(self, stream):

1413 """Run toaster on particular stream and data. 1414 Used as helper function. 1415 """ 1416 # inspect the file name 1417 if not self.inspect_filename(stream.name): 1418 self.msg("=== %s (skipped) ===" % stream.name) 1419 return 1420 1421 # check if file exists 1422 if self.options["resume"]: 1423 if self.spellclass.get_toast_stream( 1424 self, stream.name, test_exists=True): 1425 self.msg("=== %s (already done) ===" % stream.name) 1426 return 1427 1428 data = self.FILEFORMAT.Data() 1429 1430 self.msgblockbegin("=== %s ===" % stream.name) 1431 try: 1432 # inspect the file (reads only the header) 1433 data.inspect(stream) 1434 1435 # create spell instance 1436 spell = self.spellclass(toaster=self, data=data, stream=stream) 1437 1438 # inspect the spell instance 1439 if spell._datainspect() and spell.datainspect(): 1440 # read the full file 1441 data.read(stream) 1442 1443 # cast the spell on the data tree 1444 spell.recurse() 1445 1446 # save file back to disk if not readonly and the spell 1447 # changed the file 1448 if (not self.spellclass.READONLY) and spell.changed: 1449 if self.options["createpatch"]: 1450 self.writepatch(stream, data) 1451 else: 1452 self.write(stream, data) 1453 1454 except Exception: 1455 self.logger.error("TEST FAILED ON %s" % stream.name) 1456 self.logger.error( 1457 "If you were running a spell that came with PyFFI, then") 1458 self.logger.error( 1459 "please report this as a bug (include the file) on") 1460 self.logger.error( 1461 "http://sourceforge.net/tracker/?group_id=199269") 1462 # if raising test errors, reraise the exception 1463 if self.options["raisetesterror"]: 1464 raise 1465 finally: 1466 self.msgblockend()

1467

1468 - def get_toast_head_root_ext(self, filename):

1469 """Get the name of where the input file *filename* would 1470 be written to by the toaster: head, root, and extension. 1471 1472 :param filename: The name of the hypothetical file to be 1473 toasted. 1474 :type filename: :class:`str` 1475 :return: The head, root, and extension of the destination, or 1476 ``(None, None, None)`` if ``--dryrun`` is specified. 1477 :rtype: :class:`tuple` of three :class:`str`\ s 1478 """ 1479 # first cover trivial case 1480 if self.options["dryrun"]: 1481 return (None, None, None) 1482 # split original file up 1483 head, tail = os.path.split(filename) 1484 root, ext = os.path.splitext(tail) 1485 # check if head sourcedir needs replacing by destdir 1486 # and do some sanity checks if this is the case 1487 if self.options["destdir"]: 1488 if not self.options["sourcedir"]: 1489 raise ValueError( 1490 "--dest-dir specified without --source-dir") 1491 if not head.startswith(self.options["sourcedir"]): 1492 raise ValueError( 1493 "invalid --source-dir: %s does not start with %s" 1494 % (filename, self.options["sourcedir"])) 1495 head = head.replace( 1496 self.options["sourcedir"], self.options["destdir"], 1) 1497 return ( 1498 head, 1499 self.options["prefix"] + root + self.options["suffix"], 1500 ext, 1501 )

1502

1503 - def get_toast_stream(self, filename, test_exists=False):

1504 """Calls :meth:`get_toast_head_root_ext(filename)` 1505 to determine the name of the toast file, and return 1506 a stream for writing accordingly. 1507 1508 Then return a stream where result can be written to, or 1509 in case test_exists is True, test if result would overwrite a 1510 file. More specifically, if test_exists is True, then no 1511 streams are created, and True is returned if the file 1512 already exists, and False is returned otherwise. 1513 """ 1514 if self.options["dryrun"]: 1515 if test_exists: 1516 return False # temporary file never exists 1517 else: 1518 self.msg("writing to temporary file") 1519 return tempfile.TemporaryFile() 1520 head, root, ext = self.get_toast_head_root_ext(filename) 1521 if not os.path.exists(head): 1522 if test_exists: 1523 # path does not exist, so file definitely does 1524 # not exist 1525 return False 1526 else: 1527 self.logger.info("creating destination path %s" % head) 1528 os.makedirs(head) 1529 filename = os.path.join(head, root + ext) 1530 if test_exists: 1531 return os.path.exists(filename) 1532 else: 1533 if os.path.exists(filename): 1534 self.msg("overwriting %s" % filename) 1535 else: 1536 self.msg("writing %s" % filename) 1537 return open(filename, "wb")

1538

1539 - def write(self, stream, data):

1540 """Writes the data to data and raises an exception if the 1541 write fails, but restores file if fails on overwrite. 1542 """ 1543 outstream = self.spellclass.get_toast_stream(self, stream.name) 1544 if stream is outstream: 1545 # make backup 1546 stream.seek(0) 1547 backup = stream.read(-1) 1548 stream.seek(0) 1549 try: 1550 try: 1551 data.write(outstream) 1552 except: # not just Exception, also CTRL-C 1553 self.msg("write failed!!!") 1554 if stream is outstream: 1555 self.msg("attempting to restore original file...") 1556 stream.seek(0) 1557 stream.write(backup) 1558 stream.truncate() 1559 else: 1560 outstream_name = outstream.name 1561 self.msg("removing incompletely written file...") 1562 outstream.close() 1563 # temporary streams are removed on close 1564 # so check if it exists before removing 1565 if os.path.exists(outstream_name): 1566 os.remove(outstream_name) 1567 raise 1568 if stream is outstream: 1569 stream.truncate() 1570 finally: 1571 outstream.close()

1572

1573 - def writepatch(self, stream, data):

1574 """Creates a binary patch for the updated file.""" 1575 diffcmd = self.options.get('diffcmd') 1576 if not diffcmd: 1577 raise ValueError("must specify a diff command") 1578 1579 1580 # create a temporary file that won't get deleted when closed 1581 self.options["suffix"] = ".tmp" 1582 newfile = self.spellclass.get_toast_stream(self, stream.name) 1583 try: 1584 data.write(newfile) 1585 except: # not just Exception, also CTRL-C 1586 self.msg("write failed!!!") 1587 raise 1588 # use external diff command 1589 oldfile = stream 1590 oldfilename = oldfile.name 1591 patchfilename = newfile.name[:-4] + ".patch" 1592 # close all files before calling external command 1593 oldfile.close() 1594 newfile.close() 1595 self.msg("calling %s" % diffcmd) 1596 subprocess.call([diffcmd, oldfilename, newfilename, patchfilename]) 1597 # delete temporary file 1598 os.remove(newfilename)

1599 1600 if __name__ == '__main__': 1601 import doctest 1602 doctest.testmod() 1603

Source Code for Package pyffi.spells