Package cheesecake :: Module cheesecake_index
[hide private]
[frames] | no frames]

Source Code for Module cheesecake.cheesecake_index

   1  #!/usr/bin/env python 
   2  """Cheesecake: How tasty is your code? 
   3   
   4  The idea of the Cheesecake project is to rank Python packages based on various  
   5  empirical "kwalitee" factors, such as: 
   6   
   7   * whether the package can be downloaded from PyPI given its name 
   8   * whether the package can be unpacked 
   9   * whether the package can be installed into an alternate directory 
  10   * existence of certain files such as README, INSTALL, LICENSE, setup.py etc. 
  11   * percentage of modules/functions/classes/methods with docstrings 
  12   * ... and many others 
  13  """ 
  14   
  15  import os 
  16  import re 
  17  import shutil 
  18  import sys 
  19  import tempfile 
  20   
  21  from optparse import OptionParser 
  22  from urllib import urlretrieve 
  23  from urlparse import urlparse 
  24  from math import ceil 
  25   
  26  import logger 
  27   
  28  from util import pad_with_dots, pad_left_spaces, pad_right_spaces, pad_msg, pad_line 
  29  from util import run_cmd, command_successful 
  30  from util import unzip_package, untar_package, unegg_package 
  31  from util import mkdirs 
  32  from util import StdoutRedirector 
  33  from util import time_function 
  34  from codeparser import CodeParser 
  35  from cheesecake import __version__ as VERSION 
  36   
  37  __docformat__ = 'reStructuredText en' 
  38   
  39   
  40  ################################################################################ 
  41  ## Helpers. 
  42  ################################################################################ 
  43   
  44  if 'sorted' not in dir(__builtins__): 
  45      def sorted(L): 
  46          new_list = L[:] 
  47          new_list.sort() 
  48          return new_list 
49 50 if 'set' not in dir(__builtins__): 51 from sets import Set as set 52
53 -def isiterable(obj):
54 """Check whether object is iterable. 55 56 >>> isiterable([1,2,3]) 57 True 58 >>> isiterable("string") 59 True 60 >>> isiterable(object) 61 False 62 """ 63 return hasattr(obj, '__iter__') or isinstance(obj, basestring)
64
65 -def has_extension(filename, ext):
66 """Check if filename has given extension. 67 68 >>> has_extension("foobar.py", ".py") 69 True 70 >>> has_extension("foo.bar.py", ".py") 71 True 72 >>> has_extension("foobar.pyc", ".py") 73 False 74 75 This function is case insensitive. 76 >>> has_extension("FOOBAR.PY", ".py") 77 True 78 """ 79 return os.path.splitext(filename.lower())[1] == ext.lower()
80
81 -def discover_file_type(filename):
82 """Discover type of a file according to its name and its parent directory. 83 84 Currently supported file types: 85 * pyc 86 * pyo 87 * module: .py files of an application 88 * demo: .py files for documentation/demonstration purposes 89 * test: .py files used for testing 90 * special: .py file for special purposes 91 92 :Note: This function only check file's name, and doesn't touch the 93 filesystem. If you have to, check if file exists by yourself. 94 95 >>> discover_file_type('module.py') 96 'module' 97 >>> discover_file_type('./setup.py') 98 'special' 99 >>> discover_file_type('some/directory/junk.pyc') 100 'pyc' 101 >>> discover_file_type('examples/readme.txt') 102 >>> discover_file_type('examples/runthis.py') 103 'demo' 104 >>> discover_file_type('optimized.pyo') 105 'pyo' 106 107 >>> test_files = ['ut/test_this_and_that.py', 108 ... 'another_test.py', 109 ... 'TEST_MY_MODULE.PY'] 110 >>> for filename in test_files: 111 ... assert discover_file_type(filename) == 'test', filename 112 113 >>> discover_file_type('this_is_not_a_test_really.py') 114 'module' 115 """ 116 dirs = filename.split(os.path.sep) 117 dirs, filename = dirs[:-1], dirs[-1] 118 119 if filename in ["setup.py", "ez_setup.py", "__pkginfo__.py"]: 120 return 'special' 121 122 if has_extension(filename, ".pyc"): 123 return 'pyc' 124 if has_extension(filename, ".pyo"): 125 return 'pyo' 126 if has_extension(filename, ".py"): 127 for dir in dirs: 128 if dir in ['test', 'tests']: 129 return 'test' 130 elif dir in ['doc', 'docs', 'demo', 'example', 'examples']: 131 return 'demo' 132 133 # Most test frameworks look for files starting with "test_". 134 # py.test also looks at files with trailing "_test". 135 if filename.lower().startswith('test_') or \ 136 os.path.splitext(filename)[0].lower().endswith('_test'): 137 return 'test' 138 139 return 'module'
140
141 -def get_files_of_type(file_list, file_type):
142 """Return files from `file_list` that match given `file_type`. 143 144 >>> file_list = ['test/test_foo.py', 'setup.py', 'README', 'test/test_bar.py'] 145 >>> get_files_of_type(file_list, 'test') 146 ['test/test_foo.py', 'test/test_bar.py'] 147 """ 148 return filter(lambda x: discover_file_type(x) == file_type, file_list)
149
150 -def get_package_name_from_path(path):
151 """Get package name as file portion of path. 152 153 >>> get_package_name_from_path('/some/random/path/package.tar.gz') 154 'package.tar.gz' 155 >>> get_package_name_from_path('/path/underscored_name.zip') 156 'underscored_name.zip' 157 >>> get_package_name_from_path('/path/unknown.extension.txt') 158 'unknown.extension.txt' 159 """ 160 dir, filename = os.path.split(path) 161 return filename
162
163 -def get_package_name_from_url(url):
164 """Use ``urlparse`` to obtain package name from URL. 165 166 >>> get_package_name_from_url('http://www.example.com/file.tar.bz2') 167 'file.tar.bz2' 168 >>> get_package_name_from_url('https://www.example.com/some/dir/file.txt') 169 'file.txt' 170 """ 171 (scheme,location,path,param,query,fragment_id) = urlparse(url) 172 return get_package_name_from_path(path)
173
174 -def get_package_name_and_type(package, known_extensions):
175 """Return package name and type. 176 177 Package type must exists in known_extensions list. Otherwise None is 178 returned. 179 180 >>> extensions = ['tar.gz', 'zip'] 181 >>> get_package_name_and_type('underscored_name.zip', extensions) 182 ('underscored_name', 'zip') 183 >>> get_package_name_and_type('unknown.extension.txt', extensions) 184 """ 185 for package_type in known_extensions: 186 if package.endswith('.'+package_type): 187 # Package name is name of package without file extension (ex. twill-7.3). 188 return package[:package.rfind('.'+package_type)], package_type
189
190 -def get_method_arguments(method):
191 """Return tuple of arguments for given method, excluding self. 192 193 >>> class Class: 194 ... def method(s, arg1, arg2, other_arg): 195 ... pass 196 >>> get_method_arguments(Class.method) 197 ('arg1', 'arg2', 'other_arg') 198 """ 199 return method.func_code.co_varnames[1:method.func_code.co_argcount]
200
201 -def get_attributes(obj, names):
202 """Return attributes dictionary with keys from `names`. 203 204 Object is queried for each attribute name, if it doesn't have this 205 attribute, default value None will be returned. 206 207 >>> class Class: 208 ... pass 209 >>> obj = Class() 210 >>> obj.attr = True 211 >>> obj.value = 13 212 >>> obj.string = "Hello" 213 214 >>> d = get_attributes(obj, ['attr', 'string', 'other']) 215 >>> d == {'attr': True, 'string': "Hello", 'other': None} 216 True 217 """ 218 attrs = {} 219 220 for name in names: 221 attrs[name] = getattr(obj, name, None) 222 223 return attrs
224
225 -def camel2underscore(name):
226 """Convert name from CamelCase to underscore_name. 227 228 >>> camel2underscore('CamelCase') 229 'camel_case' 230 >>> camel2underscore('already_underscore_name') 231 'already_underscore_name' 232 >>> camel2underscore('BigHTMLClass') 233 'big_html_class' 234 >>> camel2underscore('') 235 '' 236 """ 237 if name and name[0].upper: 238 name = name[0].lower() + name[1:] 239
240 - def capitalize(match):
241 string = match.group(1).lower().capitalize() 242 return string[:-1] + string[-1].upper()
243
244 - def underscore(match):
245 return '_' + match.group(1).lower()
246 247 name = re.sub(r'([A-Z]+)', capitalize, name) 248 return re.sub(r'([A-Z])', underscore, name)
249
250 -def index_class_to_name(clsname):
251 """Covert index class name to index name. 252 253 >>> index_class_to_name("IndexDownload") 254 'download' 255 >>> index_class_to_name("IndexUnitTests") 256 'unit_tests' 257 >>> index_class_to_name("IndexPyPIDownload") 258 'py_pi_download' 259 """ 260 return camel2underscore(clsname.replace('Index', '', 1))
261
262 -def is_empty(path):
263 """Returns True if file or directory pointed by `path` is empty. 264 """ 265 if os.path.isfile(path) and os.path.getsize(path) == 0: 266 return True 267 if os.path.isdir(path) and os.listdir(path) == []: 268 return True 269 270 return False
271
272 -def strip_dir_part(path, root):
273 """Strip `root` part from `path`. 274 275 >>> strip_dir_part('/home/ruby/file', '/home') 276 'ruby/file' 277 >>> strip_dir_part('/home/ruby/file', '/home/') 278 'ruby/file' 279 >>> strip_dir_part('/home/ruby/', '/home') 280 'ruby/' 281 >>> strip_dir_part('/home/ruby/', '/home/') 282 'ruby/' 283 """ 284 path = path.replace(root, '', 1) 285 286 if path.startswith(os.path.sep): 287 path = path[1:] 288 289 return path
290
291 -def get_files_dirs_list(root):
292 """Return list of all files and directories below `root`. 293 294 Root directory is excluded from files/directories paths. 295 """ 296 files = [] 297 directories = [] 298 299 for dirpath, dirnames, filenames in os.walk(root): 300 dirpath = strip_dir_part(dirpath, root) 301 files.extend(map(lambda x: os.path.join(dirpath, x), filenames)) 302 directories.extend(map(lambda x: os.path.join(dirpath, x), dirnames)) 303 304 return files, directories
305
306 -def length(L):
307 """Overall length of all strings in list. 308 309 >>> length(['a', 'bc', 'd', '', 'efg']) 310 7 311 """ 312 return sum(map(lambda x: len(x), L))
313
314 -def generate_arguments(arguments, max_length):
315 """Pass list of strings in chunks of size not greater than max_length. 316 317 >>> for x in generate_arguments(['abc', 'def'], 4): 318 ... print x 319 ['abc'] 320 ['def'] 321 322 >>> for x in generate_arguments(['a', 'bc', 'd', 'e', 'f'], 2): 323 ... print x 324 ['a'] 325 ['bc'] 326 ['d', 'e'] 327 ['f'] 328 329 If a single argument is larger than max_length, ValueError is raised. 330 >>> L = [] 331 >>> for x in generate_arguments(['abc', 'de', 'fghijk', 'l'], 4): 332 ... L.append(x) 333 Traceback (most recent call last): 334 ... 335 ValueError: Argument 'fghijk' larger than 4. 336 >>> L 337 [['abc'], ['de']] 338 """ 339 L = [] 340 i = 0 341 342 # We have to look ahead, so C-style loop here. 343 while arguments: 344 if L == [] and len(arguments[i]) > max_length: 345 raise ValueError("Argument '%s' larger than %d." % (arguments[i], max_length)) 346 347 L.append(arguments[i]) 348 349 # End of arguments: yield then terminate. 350 if i == len(arguments) - 1: 351 yield L 352 break 353 354 # Adding next argument would exceed max_length, so yield now. 355 if length(L) + len(arguments[i+1]) > max_length: 356 yield L 357 L = [] 358 359 i += 1
360 361 ################################################################################ 362 ## Main index class. 363 ################################################################################ 364
365 -class NameSetter(type):
366 - def __init__(cls, name, bases, dict):
367 if 'name' not in dict: 368 setattr(cls, 'name', name) 369 370 if 'compute_with' in dict: 371 orig_compute_with = cls.compute_with 372 373 def _timed_compute_with(self, cheesecake): 374 (ret, self.time_taken) = time_function(lambda: orig_compute_with(self, cheesecake)) 375 self.cheesecake.log.debug("Index %s computed in %.2f seconds." % (self.name, self.time_taken)) 376 return ret
377 378 setattr(cls, 'compute_with', _timed_compute_with)
379
380 - def __repr__(cls):
381 return '<Index class: %s>' % cls.name
382
383 -def make_indices_dict(indices):
384 indices_dict = {} 385 for index in indices: 386 indices_dict[index.name] = index 387 return indices_dict
388
389 -class Index(object):
390 """Class describing one index. 391 392 Use it as a container index or subclass to create custom indices. 393 394 During class initialization, special attribute `name` is magically 395 set based on class name. See `NameSetter` definitions for details. 396 """ 397 __metaclass__ = NameSetter 398 399 subindices = None 400 401 name = "unnamed" 402 value = -1 403 details = "" 404 info = "" 405
406 - def __init__(self, *indices):
407 # When indices are given explicitly they override the default. 408 if indices: 409 self.subindices = [] 410 self._indices_dict = {} 411 for index in indices: 412 self.add_subindex(index) 413 else: 414 if self.subindices: 415 new_subindices = [] 416 for index in self.subindices: 417 # index must be a class subclassing from Index. 418 assert isinstance(index, type) 419 assert issubclass(index, Index) 420 new_subindices.append(index()) 421 self.subindices = new_subindices 422 else: 423 self.subindices = [] 424 # Create dictionary for fast reference. 425 self._indices_dict = make_indices_dict(self.subindices) 426 427 self._compute_arguments = get_method_arguments(self.compute)
428
429 - def _iter_indices(self):
430 """Iterate over each subindex and yield their values. 431 """ 432 for index in self.subindices: 433 # Pass Cheesecake instance to other indices. 434 yield index.compute_with(self.cheesecake) 435 # Print index info after computing. 436 if not self.cheesecake.quiet: 437 index.print_info()
438
439 - def compute_with(self, cheesecake):
440 """Take given Cheesecake instance and compute index value. 441 """ 442 self.cheesecake = cheesecake 443 return self.compute(**get_attributes(cheesecake, self._compute_arguments))
444
445 - def compute(self):
446 """Compute index value and return it. 447 448 By default this method computes sum of all subindices. Override this 449 method when subclassing for different behaviour. 450 451 Parameters to this function are dynamically prepared with use of 452 `get_attributes` function. 453 454 :Warning: Don't use \*args and \*\*kwds arguments for this method. 455 """ 456 self.value = sum(self._iter_indices()) 457 return self.value
458
459 - def decide(self, cheesecake, when):
460 """Decide if this index should be computed. 461 462 If index has children, it will automatically remove all for which 463 decide() return false. 464 """ 465 if self.subindices: 466 # Iterate over copy, as we may remove some elements. 467 for index in self.subindices[:]: 468 if not getattr(index, 'decide_' + when)(cheesecake): 469 self.remove_subindex(index.name) 470 return self.subindices 471 return True
472
473 - def decide_before_download(self, cheesecake):
474 return self.decide(cheesecake, 'before_download')
475
476 - def decide_after_download(self, cheesecake):
477 return self.decide(cheesecake, 'after_download')
478
479 - def add_info(self, info_line):
480 """Add information about index computation process, which will 481 be visible with --verbose flag. 482 """ 483 self.info += "[%s] %s\n" % (index_class_to_name(self.name), info_line)
484
485 - def _get_max_value(self):
486 if self.subindices: 487 return sum(map(lambda index: index.max_value, 488 self.subindices)) 489 return 0
490 491 max_value = property(_get_max_value) 492
493 - def _get_requirements(self):
494 if self.subindices: 495 return list(self._compute_arguments) + \ 496 reduce(lambda x,y: x + y.requirements, self.subindices, []) 497 return list(self._compute_arguments)
498 499 requirements = property(_get_requirements) 500
501 - def add_subindex(self, index):
502 """Add subindex. 503 504 :Parameters: 505 `index` : Index instance 506 Index instance for inclusion. 507 """ 508 if not isinstance(index, Index): 509 raise ValueError("subindex have to be instance of Index") 510 511 self.subindices.append(index) 512 self._indices_dict[index.name] = index
513
514 - def remove_subindex(self, index_name):
515 """Remove subindex (refered by name). 516 517 :Parameters: 518 `index` : Index name 519 Index name to be removed. 520 """ 521 index = self._indices_dict[index_name] 522 self.subindices.remove(index) 523 del self._indices_dict[index_name]
524
525 - def _print_info_one(self):
526 if self.cheesecake.verbose: 527 sys.stdout.write(self.get_info()) 528 print "%s (%s)" % (pad_msg(index_class_to_name(self.name), self.value), self.details)
529
530 - def _print_info_many(self):
531 max_value = self.max_value 532 if max_value == 0: 533 return 534 535 percentage = int(ceil(float(self.value) / float(max_value) * 100)) 536 print pad_line("-") 537 538 print pad_msg("%s INDEX (ABSOLUTE)" % self.name, self.value) 539 msg = pad_msg("%s INDEX (RELATIVE)" % self.name, percentage) 540 msg += " (%d out of a maximum of %d points is %d%%)" %\ 541 (self.value, max_value, percentage) 542 543 print msg 544 print
545
546 - def print_info(self):
547 """Print index name padded with dots, followed by value and details. 548 """ 549 if self.subindices: 550 self._print_info_many() 551 else: 552 self._print_info_one()
553
554 - def __getitem__(self, name):
555 return self._indices_dict[name]
556
557 - def get_info(self):
558 if self.subindices: 559 return ''.join(map(lambda index: index.get_info(), self.subindices)) 560 return self.info
561 562 ################################################################################ 563 ## Index that computes scores based on files and directories. 564 ################################################################################ 565
566 -class OneOf(object):
567 - def __init__(self, *possibilities):
568 self.possibilities = possibilities
569 - def __str__(self):
570 return '/'.join(map(lambda x: str(x), self.possibilities))
571
572 -def WithOptionalExt(name, extensions):
573 """Handy way of writing Cheese rules for files with extensions. 574 575 Instead of writing: 576 >>> one_of = OneOf('readme', 'readme.html', 'readme.txt') 577 578 Write this: 579 >>> opt_ext = WithOptionalExt('readme', ['html', 'txt']) 580 581 It means the same! (representation have a meaning) 582 >>> str(one_of) == str(opt_ext) 583 True 584 """ 585 possibilities = [name] 586 possibilities.extend(map(lambda x: name + '.' + x, extensions)) 587 588 return OneOf(*possibilities)
589
590 -def Doc(name):
591 return WithOptionalExt(name, ['html', 'txt'])
592
593 -class FilesIndex(Index):
594 _used_rules = [] 595
596 - def _compute_from_rules(self, files_list, package_dir, files_rules):
597 self._used_rules = [] 598 files_count = 0 599 value = 0 600 601 for filename in files_list: 602 if not is_empty(os.path.join(package_dir, filename)): 603 score = self.get_score(os.path.basename(filename), files_rules) 604 if score != 0: 605 value += score 606 files_count += 1 607 608 return files_count, value
609
610 - def get_score(self, name, specs):
611 for entry, value in specs.iteritems(): 612 if self.match_filename(name, entry): 613 self.cheesecake.log.debug("%d points entry found: %s (%s)" % \ 614 (value, name, entry)) 615 return value 616 617 return 0
618
619 - def get_not_used(self, files_rules):
620 """Get only these of files_rules that didn't match during computation. 621 622 >>> rules = { 623 ... Doc('readme'): 30, 624 ... OneOf(Doc('license'), Doc('copying')): 30, 625 ... 'demo': 10, 626 ... } 627 >>> index = FilesIndex() 628 >>> index._used_rules.append('demo') 629 >>> map(lambda x: str(x), index.get_not_used(rules.keys())) 630 ['license/license.html/license.txt/copying/copying.html/copying.txt', 'readme/readme.html/readme.txt'] 631 """ 632 return filter(lambda rule: rule not in self._used_rules, 633 files_rules)
634
635 - def match_filename(self, name, rule):
636 """Check if `name` matches given `rule`. 637 """
638 - def equal(x, y):
639 x_root, x_ext = os.path.splitext(x) 640 y_root, y_ext = os.path.splitext(y.lower()) 641 if x_root in [y_root.lower(), y_root.upper(), y_root.capitalize()] \ 642 and x_ext in [y_ext.lower(), y_ext.upper()]: 643 return True 644 return False
645 646 if rule in self._used_rules: 647 return False 648 649 if isinstance(rule, basestring): 650 if equal(name, rule): 651 self._used_rules.append(rule) 652 return True 653 elif isinstance(rule, OneOf): 654 for poss in rule.possibilities: 655 if self.match_filename(name, poss): 656 self._used_rules.append(rule) 657 return True 658 659 return False
660 661 ################################################################################ 662 ## Installability index. 663 ################################################################################ 664
665 -class IndexUrlDownload(Index):
666 """Give points for successful downloading of a package. 667 """ 668 max_value = 25 669
670 - def compute(self, downloaded_from_url, package, url):
671 if downloaded_from_url: 672 self.details = "downloaded package %s from URL %s" % (package, url) 673 self.value = self.max_value 674 else: 675 self.value = 0 676 677 return self.value
678
679 - def decide_before_download(self, cheesecake):
680 return cheesecake.url
681
682 -class IndexUnpack(Index):
683 """Give points for successful unpacking of a package archive. 684 """ 685 max_value = 25 686
687 - def compute(self, unpacked):
688 if unpacked: 689 self.details = "package unpacked successfully" 690 self.value = self.max_value 691 else: 692 self.details = "package couldn't be unpacked" 693 self.value = 0 694 695 return self.value
696
697 -class IndexUnpackDir(Index):
698 """Check if package unpack directory resembles package archive name. 699 """ 700 max_value = 15 701
702 - def compute(self, unpack_dir, original_package_name):
703 self.details = "unpack directory is " + unpack_dir 704 705 if original_package_name: 706 self.details += " instead of the expected " + original_package_name 707 self.value = 0 708 else: 709 self.details += " as expected" 710 self.value = self.max_value 711 712 return self.value
713
714 - def decide_after_download(self, cheesecake):
715 return cheesecake.package_type != 'egg'
716
717 -class IndexSetupPy(FilesIndex):
718 """Reward packages that have setup.py file. 719 """ 720 name = "setup.py" 721 max_value = 25 722 723 files_rules = { 724 'setup.py': 25, 725 } 726
727 - def compute(self, files_list, package_dir):
728 setup_py_found, self.value = self._compute_from_rules(files_list, package_dir, self.files_rules) 729 730 if setup_py_found: 731 self.details = "setup.py found" 732 else: 733 self.details = "setup.py not found" 734 735 return self.value
736
737 - def decide_after_download(self, cheesecake):
738 return cheesecake.package_type != 'egg'
739
740 -class IndexInstall(Index):
741 """Check if package can be installed via "python setup.py" command. 742 """ 743 max_value = 50 744
745 - def compute(self, installed, sandbox_install_dir):
746 if installed: 747 self.details = "package installed in %s" % sandbox_install_dir 748 self.value = self.max_value 749 else: 750 self.details = "could not install package in %s" % sandbox_install_dir 751 self.value = 0 752 753 return self.value
754
755 - def decide_before_download(self, cheesecake):
756 return not cheesecake.static_only
757
758 -class IndexPyPIDownload(Index):
759 """Check if package was successfully downloaded from PyPI 760 and how far from it actual package was. 761 762 Distance is number of links user have to follow to download 763 a given software package. 764 """ 765 max_value = 50 766 distance_penalty = -5 767
768 - def compute(self, package, found_on_cheeseshop, found_locally, distance_from_pypi, download_url):
769 if download_url: 770 self.value = self.max_value 771 772 self.details = "downloaded package " + package 773 774 if not found_on_cheeseshop: 775 self.value += (distance_from_pypi - 1) * self.distance_penalty 776 777 if distance_from_pypi: 778 self.details += " following %d link" % distance_from_pypi 779 if distance_from_pypi > 1: 780 self.details += "s" 781 self.details += " from PyPI" 782 else: 783 self.details += " from " + download_url 784 else: 785 self.details += " directly from the Cheese Shop" 786 else: 787 if found_locally: 788 self.details = "found on local filesystem" 789 self.value = 0 790 791 return self.value
792
793 - def decide_before_download(self, cheesecake):
794 return cheesecake.name
795
796 -class IndexGeneratedFiles(Index):
797 """Lower score for automatically generated files that should 798 not be present in a package. 799 """ 800 generated_files_penalty = -20 801 max_value = 0 802
803 - def compute(self, files_list):
804 self.value = 0 805 806 pyc_files = len(get_files_of_type(files_list, 'pyc')) 807 pyo_files = len(get_files_of_type(files_list, 'pyo')) 808 809 if pyc_files > 0 or pyo_files > 0: 810 self.value += self.generated_files_penalty 811 812 self.details = "%d .pyc and %d .pyo files found" % \ 813 (pyc_files, pyo_files) 814 815 return self.value
816
817 - def decide_after_download(self, cheesecake):
818 return cheesecake.package_type != 'egg'
819
820 -class IndexInstallability(Index):
821 name = "INSTALLABILITY" 822 823 subindices = [ 824 IndexPyPIDownload, 825 IndexUrlDownload, 826 IndexUnpack, 827 IndexUnpackDir, 828 IndexSetupPy, 829 IndexInstall, 830 IndexGeneratedFiles, 831 ]
832 833 ################################################################################ 834 ## Documentation index. 835 ################################################################################ 836
837 -class IndexRequiredFiles(FilesIndex):
838 """Check for existence of important files, like README or INSTALL. 839 """ 840 cheese_files = { 841 Doc('readme'): 30, 842 OneOf(Doc('license'), Doc('copying')): 30, 843 844 OneOf(Doc('announce'), Doc('changelog'), Doc('changes')): 20, 845 Doc('install'): 20, 846 847 Doc('authors'): 10, 848 Doc('faq'): 10, 849 Doc('news'): 10, 850 Doc('thanks'): 10, 851 Doc('todo'): 10, 852 } 853 854 cheese_dirs = { 855 OneOf('doc', 'docs'): 30, 856 OneOf('test', 'tests'): 30, 857 858 OneOf('demo', 'example', 'examples'): 10, 859 } 860 861 max_value = sum(cheese_files.values() + cheese_dirs.values()) 862
863 - def compute(self, files_list, dirs_list, package_dir):
864 # Inform user of files and directories the package is missing.
865 - def make_info(dictionary, what):
866 missing = self.get_not_used(dictionary.keys()) 867 importance = {30: ' critical', 20: ' important'} 868 869 positive_msg = "Package has%s %s: %s." 870 negative_msg = "Package doesn't have%s %s: %s." 871 872 for key in dictionary.keys(): 873 msg = positive_msg 874 if key in missing: 875 msg = negative_msg 876 self.add_info(msg % (importance.get(dictionary[key], ''), what, str(key)))
877 878 # Compute required files. 879 files_count, files_value = self._compute_from_rules(files_list, package_dir, self.cheese_files) 880 make_info(self.cheese_files, 'file') 881 882 # Compute required directories. 883 dirs_count, dirs_value = self._compute_from_rules(dirs_list, package_dir, self.cheese_dirs) 884 make_info(self.cheese_dirs, 'directory') 885 886 self.value = files_value + dirs_value 887 888 self.details = "%d files and %d required directories found" % \ 889 (files_count, dirs_count) 890 891 return self.value
892
893 -class IndexDocstrings(Index):
894 """Compute how many objects have relevant docstrings. 895 """ 896 max_value = 100 897
898 - def compute(self, object_cnt, docstring_cnt):
899 percent = 0 900 if object_cnt > 0: 901 percent = float(docstring_cnt)/float(object_cnt) 902 903 # Scale the result. 904 self.value = int(ceil(percent * self.max_value)) 905 906 self.details = "found %d/%d=%.2f%% objects with docstrings" %\ 907 (docstring_cnt, object_cnt, percent*100) 908 909 return self.value
910
911 -class IndexFormattedDocstrings(Index):
912 """Compute how many of existing docstrings include any formatting, 913 like epytext or reST. 914 """ 915 max_value = 30 916
917 - def compute(self, object_cnt, docformat_cnt):
918 percent = 0 919 if object_cnt > 0: 920 percent = float(docformat_cnt)/float(object_cnt) 921 922 # Scale the result. 923 # We give 10p for 25% of formatted docstrings, 20p for 50% and 30p for 75%. 924 self.value = 0 925 if percent > 0.75: 926 self.add_info("%.2f%% formatted docstrings found, which is > 75%% and is worth 30p." % (percent*100)) 927 self.value = 30 928 elif percent > 0.50: 929 self.add_info("%.2f%% formatted docstrings found, which is > 50%% and is worth 20p." % (percent*100)) 930 self.value = 20 931 elif percent > 0.25: 932 self.add_info("%.2f%% formatted docstrings found, which is > 25%% and is worth 10p." % (percent*100)) 933 self.value = 10 934 else: 935 self.add_info("%.2f%% formatted docstrings found, which is < 25%%, no points given." % (percent*100)) 936 937 self.details = "found %d/%d=%.2f%% objects with formatted docstrings" %\ 938 (docformat_cnt, object_cnt, percent*100) 939 940 return self.value
941
942 -class IndexDocumentation(Index):
943 name = "DOCUMENTATION" 944 945 subindices = [ 946 IndexRequiredFiles, 947 IndexDocstrings, 948 IndexFormattedDocstrings, 949 ]
950 951 ################################################################################ 952 ## Code "kwalitee" index. 953 ################################################################################ 954
955 -class IndexUnitTests(Index):
956 """Compute unittest index as percentage of methods/functions 957 that are exercised in unit tests. 958 """ 959 max_value = 50 960
961 - def compute(self, files_list, functions, classes, package_dir):
962 unittest_cnt = 0 963 functions_tested = set() 964 965 # Gather all function names called from test files. 966 for testfile in get_files_of_type(files_list, 'test'): 967 fullpath = os.path.join(package_dir, testfile) 968 code = CodeParser(fullpath, self.cheesecake.log.debug) 969 970 functions_tested = functions_tested.union(code.functions_called) 971 972 for name in functions + classes: 973 if name in functions_tested: 974 unittest_cnt += 1 975 self.cheesecake.log.debug("%s is unit tested" % name) 976 977 functions_classes_cnt = len(functions) + len(classes) 978 percent = 0 979 if functions_classes_cnt > 0: 980 percent = float(unittest_cnt)/float(functions_classes_cnt) 981 982 # Scale the result. 983 self.value = int(ceil(percent * self.max_value)) 984 985 self.details = "found %d/%d=%.2f%% unit tested classes/methods/functions." %\ 986 (unittest_cnt, functions_classes_cnt, percent*100) 987 988 return self.value
989
990 -class IndexUnitTested(Index):
991 """Check if the package have unit tests which can be easily found by 992 any of known test frameworks. 993 """ 994 max_value = 30 995
996 - def compute(self, doctests_count, unittests_count, files_list, classes, methods):
997 unit_tested = False 998 999 if doctests_count > 0: 1000 self.add_info("Package includes doctest tests.") 1001 unit_tested = True 1002 1003 if unittests_count > 0: 1004 self.add_info("Package have tests that inherit from unittest.TestCase.") 1005 unit_tested = True 1006 1007 if get_files_of_type(files_list, 'test'): 1008 self.add_info("Package have filenames which probably contain tests (in format test_* or *_test)") 1009 unit_tested = True 1010 1011 for method in methods: 1012 if self._is_test_method(method): 1013 self.add_info("Some classes have setUp/tearDown methods which are commonly used in unit tests.") 1014 unit_tested = True 1015 break 1016 1017 if unit_tested: 1018 self.value = self.max_value 1019 self.details = "has unit tests" 1020 else: 1021 self.value = 0 1022 self.details = "don't have unit tests" 1023 1024 return self.value
1025
1026 - def _is_test_method(self, method):
1027 nose_methods = ['setup', 'setup_package', 'setup_module', 'setUp', 1028 'setUpPackage', 'setUpModule', 1029 'teardown', 'teardown_package', 'teardown_module', 1030 'tearDown', 'tearDownModule', 'tearDownPackage'] 1031 1032 for test_method in nose_methods: 1033 if method.endswith(test_method): 1034 return True 1035 return False
1036
1037 -class IndexPyLint(Index):
1038 """Compute pylint index of the whole package. 1039 """ 1040 name = "pylint" 1041 max_value = 50 1042 1043 disabled_messages = [ 1044 'W0403', # relative import 1045 'W0406', # importing of self 1046 ] 1047 pylint_args = ' '.join(map(lambda x: '--disable-msg=%s' % x, disabled_messages)) 1048
1049 - def compute(self, files_list, package_dir):
1050 # Maximum length of arguments (not very precise). 1051 max_arguments_length = 65536 1052 1053 # Exclude __init__.py files from score as they cause pylint 1054 # to fail with ImportError "Unable to find module for %s in %s". 1055 files_to_lint = filter(lambda name: not name.endswith('__init__.py'), 1056 get_files_of_type(files_list, 'module')) 1057 1058 # Switching cwd so that pylint works correctly regarding 1059 # running it on individual modules. 1060 original_cwd = os.getcwd() 1061 1062 # Note: package_dir may be a file if the archive contains a single file. 1063 # If this is the case, change dir to the parent dir of that file. 1064 if os.path.isfile(package_dir): 1065 package_dir = os.path.dirname(package_dir) 1066 1067 os.chdir(package_dir) 1068 1069 pylint_score = 0 1070 count = 0 1071 error_count = 0 1072 1073 for filenames in generate_arguments(files_to_lint, max_arguments_length - len(self.pylint_args)): 1074 filenames = ' '.join(filenames) 1075 self.cheesecake.log.debug("Running pylint on files: %s." % filenames) 1076 1077 rc, output = run_cmd("pylint %s %s" % (filenames, self.pylint_args)) 1078 if rc: 1079 self.cheesecake.log.debug("encountered an error (%d):\n***\n%s\n***\n" % (rc, output)) 1080 error_count += 1 1081 else: 1082 # Extract score from pylint output. 1083 score_line = output.split("\n")[-3] 1084 s = re.search(r" (-?\d+\.\d+)/10", score_line) 1085 if s: 1086 pylint_score += float(s.group(1)) 1087 count += 1 1088 1089 # Switching back to the original cwd. 1090 os.chdir(original_cwd) 1091 1092 if count: 1093 pylint_score = float(pylint_score)/float(count) 1094 self.details = "pylint score was %.2f out of 10" % pylint_score 1095 elif error_count: 1096 self.details = "encountered an error during pylint execution" 1097 else: 1098 self.details = "no files to check found" 1099 1100 # Assume scores below zero as zero for means of index value computation. 1101 if pylint_score < 0: 1102 pylint_score = 0 1103 self.value = int(ceil(pylint_score/10.0 * self.max_value)) 1104 1105 self.add_info("Score is %.2f/10, which is %d%% of maximum %d points = %d." % 1106 (pylint_score, int(pylint_score*10), self.max_value, self.value)) 1107 1108 return self.value
1109
1110 - def decide_before_download(self, cheesecake):
1111 # Try to run the pylint script 1112 if not command_successful("pylint --version"): 1113 cheesecake.log.debug("pylint not properly installed, omitting pylint index.") 1114 return False 1115 1116 return not cheesecake.lite
1117
1118 -class IndexCodeKwalitee(Index):
1119 name = "CODE KWALITEE" 1120 1121 subindices = [ 1122 IndexPyLint, 1123 #IndexUnitTests, 1124 IndexUnitTested, 1125 ]
1126 1127 ################################################################################ 1128 ## Main Cheesecake class. 1129 ################################################################################ 1130
1131 -class CheesecakeError(Exception):
1132 """Custom exception class for Cheesecake-specific errors. 1133 """ 1134 pass
1135 1136
1137 -class CheesecakeIndex(Index):
1138 name = "Cheesecake" 1139 subindices = [ 1140 IndexInstallability, 1141 IndexDocumentation, 1142 IndexCodeKwalitee, 1143 ]
1144 1145
1146 -class Step(object):
1147 """Single step during computation of package score. 1148 """
1149 - def __init__(self, provides):
1150 self.provides = provides
1151
1152 - def decide(self, cheesecake):
1153 """Decide if step should be run. 1154 1155 It checks if there's at least one index from current profile that need 1156 variables provided by this step. Override this method for other behaviour. 1157 """ 1158 for provide in self.provides: 1159 if provide in cheesecake.index.requirements: 1160 return True 1161 return False
1162
1163 -class StepByVariable(Step):
1164 """Step which is always run if given Cheesecake instance variable is true. 1165 """
1166 - def __init__(self, variable_name, provides):
1167 self.variable_name = variable_name 1168 Step.__init__(self, provides)
1169
1170 - def decide(self, cheesecake):
1171 if getattr(cheesecake, self.variable_name, None): 1172 return True 1173 1174 # Fallback to the default. 1175 return Step.decide(self, cheesecake)
1176
1177 -class Cheesecake(object):
1178 """Computes 'goodness' of Python packages. 1179 1180 Generates "cheesecake index" that takes into account things like: 1181 1182 * whether the package can be downloaded 1183 * whether the package can be unpacked 1184 * whether the package can be installed into an alternate directory 1185 * existence of certain files such as README, INSTALL, LICENSE, setup.py etc. 1186 * existence of certain directories such as doc, test, demo, examples 1187 * percentage of modules/functions/classes/methods with docstrings 1188 * percentage of functions/methods that are unit tested 1189 * average pylint score for all non-test and non-demo modules 1190 """ 1191 1192 steps = {} 1193 1194 package_types = { 1195 "tar.gz": untar_package, 1196 "tgz": untar_package, 1197 "zip": unzip_package, 1198 "egg": unegg_package, 1199 } 1200
1201 - def __init__(self, name="", url="", path="", sandbox=None, 1202 logfile=None, verbose=False, quiet=False, static_only=False, 1203 lite=False, keep_log=False):
1204 """Initialize critical variables, download and unpack package, 1205 walk package tree. 1206 """ 1207 self.name = name 1208 self.url = url 1209 self.package_path = path 1210 1211 if self.name: 1212 self.package = self.name 1213 elif self.url: 1214 self.package = get_package_name_from_url(self.url) 1215 elif self.package_path: 1216 self.package = get_package_name_from_path(self.package_path) 1217 else: 1218 self.raise_exception("No package name, URL or path specified... exiting") 1219 1220 # Setup a sandbox. 1221 self.sandbox = sandbox or tempfile.mkdtemp(prefix='cheesecake') 1222 if not os.path.isdir(self.sandbox): 1223 os.mkdir(self.sandbox) 1224 1225 self.verbose = verbose 1226 self.quiet = quiet 1227 self.static_only = static_only 1228 self.lite = lite 1229 self.keep_log = keep_log 1230 1231 self.sandbox_pkg_file = "" 1232 self.sandbox_pkg_dir = "" 1233 self.sandbox_install_dir = "" 1234 1235 # Configure logging as soon as possible. 1236 self.configure_logging(logfile) 1237 1238 # Setup Cheesecake index. 1239 self.index = CheesecakeIndex() 1240 1241 self.index.decide_before_download(self) 1242 self.log.debug("Profile requirements: %s." % ', '.join(sorted(self.index.requirements))) 1243 1244 # Get the package. 1245 self.run_step('get_pkg_from_pypi') 1246 self.run_step('download_pkg') 1247 self.run_step('copy_pkg') 1248 1249 # Get package name and type. 1250 name_and_type = get_package_name_and_type(self.package, self.package_types.keys()) 1251 1252 if not name_and_type: 1253 msg = "Could not determine package type for package '%s'" % self.package 1254 msg += "\nCurrently recognized types: " + ", ".join(self.package_types.keys()) 1255 self.raise_exception(msg) 1256 1257 self.package_name, self.package_type = name_and_type 1258 self.log.debug("Package name: " + self.package_name) 1259 self.log.debug("Package type: " + self.package_type) 1260 1261 # Make last indices decisions. 1262 self.index.decide_after_download(self) 1263 1264 # Unpack package and list its files. 1265 self.run_step('unpack_pkg') 1266 self.run_step('walk_pkg') 1267 1268 # Install package. 1269 self.run_step('install_pkg')
1270
1271 - def raise_exception(self, msg):
1272 """Cleanup, print error message and raise CheesecakeError. 1273 1274 Don't use logging, since it can be called before logging has been setup. 1275 """ 1276 self.cleanup(remove_log_file=False) 1277 1278 msg += "\nDetailed info available in log file %s" % self.logfile 1279 1280 raise CheesecakeError(msg)
1281
1282 - def cleanup(self, remove_log_file=True):
1283 """Delete temporary directories and files that were created 1284 in the sandbox. At the end delete the sandbox itself. 1285 """ 1286 if os.path.isfile(self.sandbox_pkg_file): 1287 self.log("Removing file %s" % self.sandbox_pkg_file) 1288 os.unlink(self.sandbox_pkg_file) 1289
1290 - def delete_dir(dirname):
1291 "Delete directory recursively and generate log message." 1292 if os.path.isdir(dirname): 1293 self.log("Removing directory %s" % dirname) 1294 shutil.rmtree(dirname)
1295 1296 delete_dir(self.sandbox) 1297 1298 if remove_log_file and not self.keep_log: 1299 os.unlink(os.path.join(self.sandbox, self.logfile))
1300
1301 - def configure_logging(self, logfile=None):
1302 """Default settings for logging. 1303 1304 If verbose, log goes to console, else it goes to logfile. 1305 log.debug and log.info goes to logfile. 1306 log.warn and log.error go to both logfile and stdout. 1307 """ 1308 if logfile: 1309 self.logfile = logfile 1310 else: 1311 self.logfile = os.path.join(tempfile.gettempdir(), self.package + ".log") 1312 1313 logger.setconsumer('logfile', open(str(self.logfile), 'w', buffering=1)) 1314 logger.setconsumer('console', logger.STDOUT) 1315 logger.setconsumer('null', None) 1316 1317 self.log = logger.MultipleProducer('cheesecake logfile') 1318 self.log.info = logger.MultipleProducer('cheesecake logfile') 1319 self.log.debug = logger.MultipleProducer('cheesecake logfile') 1320 self.log.warn = logger.MultipleProducer('cheesecake console') 1321 self.log.error = logger.MultipleProducer('cheesecake console')
1322
1323 - def run_step(self, step_name):
1324 """Run step if its decide() method returns True. 1325 """ 1326 step = self.steps[step_name] 1327 if step.decide(self): 1328 step_method = getattr(self, step_name) 1329 step_method()
1330 1331 steps['get_pkg_from_pypi'] = StepByVariable('name', 1332 ['download_url', 1333 'distance_from_pypi', 1334 'found_on_cheeseshop', 1335 'found_locally', 1336 'sandbox_pkg_file'])
1337 - def get_pkg_from_pypi(self):
1338 """Download package using setuptools utilities. 1339 1340 New attributes: 1341 download_url : str 1342 URL that package was downloaded from. 1343 distance_from_pypi : int 1344 How many hops setuptools had to make to download package. 1345 found_on_cheeseshop : bool 1346 Whenever package has been found on CheeseShop. 1347 found_locally : bool 1348 Whenever package has been already installed. 1349 """ 1350 self.log.info("Trying to download package %s from PyPI using setuptools utilities" % self.name) 1351 1352 try: 1353 from setuptools.package_index import PackageIndex 1354 from pkg_resources import Requirement 1355 from distutils import log 1356 from distutils.errors import DistutilsError 1357 except ImportError, e: 1358 msg = "Error: setuptools is not installed and is required for downloading a package by name\n" 1359 msg += "You can download and process a package by its full URL via the -u or --url option\n" 1360 msg += "Example: python cheesecake.py --url=http://www.mems-exchange.org/software/durus/Durus-3.1.tar.gz" 1361 self.raise_exception(msg) 1362
1363 - def drop_setuptools_info(stdout, error=None):
1364 """Drop all setuptools output as INFO. 1365 """ 1366 self.log.info("*** Begin setuptools output") 1367 map(self.log.info, stdout.splitlines()) 1368 if error: 1369 self.log.info(str(error)) 1370 self.log.info("*** End setuptools output")
1371
1372 - def fetch_package(mode):
1373 """Fetch package from PyPI. 1374 1375 Mode can be one of: 1376 * 'pypi_source': get source package from PyPI 1377 * 'pypi_any': get source/egg package from PyPI 1378 * 'any': get package from PyPI or local filesystem 1379 1380 Returns tuple (status, output), where `status` is True 1381 if fetch was successful and False if it failed. `output` 1382 is PackageIndex.fetch() return value. 1383 """ 1384 if 'pypi' in mode: 1385 pkgindex = PackageIndex(search_path=[]) 1386 else: 1387 pkgindex = PackageIndex() 1388 1389 if mode == 'pypi_source': 1390 source = True 1391 else: 1392 source = False 1393 1394 try: 1395 output = pkgindex.fetch(Requirement.parse(self.name), 1396 self.sandbox, 1397 force_scan=True, 1398 source=source) 1399 return True, output 1400 except DistutilsError, e: 1401 return False, e
1402 1403 # Temporarily set the log verbosity to INFO so we can capture setuptools 1404 # info messages. 1405 old_threshold = log.set_threshold(log.INFO) 1406 old_stdout = sys.stdout 1407 sys.stdout = StdoutRedirector() 1408 1409 # Try to get source package from PyPI first, then egg from PyPI, and if 1410 # that fails search in locally installed packages. 1411 for mode, info in [('pypi_source', "source package on PyPI"), 1412 ('pypi_any', "egg on PyPI"), 1413 ('any', "locally installed package")]: 1414 msg = "Looking for %s... " % info 1415 status, output = fetch_package(mode) 1416 if status and output: 1417 self.log.info(msg + "found!") 1418 break 1419 self.log.info(msg + "failed.") 1420 1421 # Bring back old stdout. 1422 captured_stdout = sys.stdout.read_buffer() 1423 sys.stdout = old_stdout 1424 log.set_threshold(old_threshold) 1425 1426 # If all runs failed, we must raise an error. 1427 if not status: 1428 drop_setuptools_info(captured_stdout, output) 1429 self.raise_exception("Error: setuptools returned an error: %s\n" % str(output).splitlines()[0]) 1430 1431 # If fetch returned nothing, package wasn't found. 1432 if output is None: 1433 drop_setuptools_info(captured_stdout) 1434 self.raise_exception("Error: Could not find distribution for " + self.name) 1435 1436 # Defaults. 1437 self.download_url = "" 1438 self.distance_from_pypi = 0 1439 self.found_on_cheeseshop = False 1440 self.found_locally = False 1441 1442 for line in captured_stdout.splitlines(): 1443 s = re.search(r"Reading http(.*)", line) 1444 if s: 1445 inspected_url = s.group(1) 1446 if not re.search(r"www.python.org\/pypi", inspected_url): 1447 self.distance_from_pypi += 1 1448 continue 1449 s = re.search(r"Downloading (.*)", line) 1450 if s: 1451 self.download_url = s.group(1) 1452 break 1453 1454 self.sandbox_pkg_file = output 1455 self.package = get_package_name_from_path(output) 1456 self.log.info("Downloaded package %s from %s" % (self.package, self.download_url)) 1457 1458 if os.path.isdir(self.sandbox_pkg_file): 1459 self.found_locally = True 1460 1461 if re.search(r"cheeseshop.python.org", self.download_url): 1462 self.found_on_cheeseshop = True
1463 1464 steps['download_pkg'] = StepByVariable('url', 1465 ['sandbox_pkg_file', 1466 'downloaded_from_url'])
1467 - def download_pkg(self):
1468 """Use ``urllib.urlretrieve`` to download package to file in sandbox dir. 1469 """ 1470 #self.log("Downloading package %s from URL %s" % (self.package, self.url)) 1471 self.sandbox_pkg_file = os.path.join(self.sandbox, self.package) 1472 try: 1473 downloaded_filename, headers = urlretrieve(self.url, self.sandbox_pkg_file) 1474 except IOError, e: 1475 self.log.error("Error downloading package %s from URL %s" % (self.package, self.url)) 1476 self.raise_exception(str(e)) 1477 #self.log("Downloaded package %s to %s" % (self.package, downloaded_filename)) 1478 1479 if headers.gettype() in ["text/html"]: 1480 f = open(downloaded_filename) 1481 if re.search("404 Not Found", "".join(f.readlines())): 1482 f.close() 1483 self.raise_exception("Got '404 Not Found' error while trying to download package ... exiting") 1484 f.close() 1485 1486 self.downloaded_from_url = True
1487 1488 steps['copy_pkg'] = StepByVariable('package_path', 1489 ['sandbox_pkg_file'])
1490 - def copy_pkg(self):
1491 """Copy package file to sandbox directory. 1492 """ 1493 self.sandbox_pkg_file = os.path.join(self.sandbox, self.package) 1494 if not os.path.isfile(self.package_path): 1495 self.raise_exception("%s is not a valid file ... exiting" % self.package_path) 1496 self.log("Copying file %s to %s" % (self.package_path, self.sandbox_pkg_file)) 1497 shutil.copyfile(self.package_path, self.sandbox_pkg_file)
1498 1499 steps['unpack_pkg'] = Step(['original_package_name', 1500 'sandbox_pkg_dir', 1501 'unpacked', 1502 'unpack_dir'])
1503 - def unpack_pkg(self):
1504 """Unpack the package in the sandbox directory. 1505 1506 Check `package_types` attribute for list of currently supported 1507 archive types. 1508 1509 New attributes: 1510 original_package_name : str 1511 Package name guessed from the package name. Will be set only 1512 if package name is different than unpacked directory name. 1513 """ 1514 self.sandbox_pkg_dir = os.path.join(self.sandbox, self.package_name) 1515 if os.path.isdir(self.sandbox_pkg_dir): 1516 shutil.rmtree(self.sandbox_pkg_dir) 1517 1518 # Call appropriate function to unpack the package. 1519 unpack = self.package_types[self.package_type] 1520 self.unpack_dir = unpack(self.sandbox_pkg_file, self.sandbox) 1521 1522 if self.unpack_dir is None: 1523 self.raise_exception("Could not unpack package %s ... exiting" % \ 1524 self.sandbox_pkg_file) 1525 1526 self.unpacked = True 1527 1528 if self.unpack_dir != self.package_name: 1529 self.original_package_name = self.package_name 1530 self.package_name = self.unpack_dir
1531 1532 steps['walk_pkg'] = Step(['dirs_list', 1533 'docstring_cnt', 1534 'docformat_cnt', 1535 'doctests_count', 1536 'unittests_count', 1537 'files_list', 1538 'functions', 1539 'classes', 1540 'methods', 1541 'object_cnt', 1542 'package_dir'])
1543 - def walk_pkg(self):
1544 """Get package files and directories. 1545 1546 New attributes: 1547 dirs_list : list 1548 List of directories package contains. 1549 docstring_cnt : int 1550 Number of docstrings found in all package objects. 1551 docformat_cnt : int 1552 Number of formatted docstrings found in all package objects. 1553 doctests_count : int 1554 Number of docstrings that include doctests. 1555 unittests_count : int 1556 Number of classes which inherit from unittest.TestCase. 1557 files_list : list 1558 List of files package contains. 1559 functions : list 1560 List of all functions defined in package sources. 1561 classes : list 1562 List of all classes defined in package sources. 1563 methods : list 1564 List of all methods defined in package sources. 1565 object_cnt : int 1566 Number of documentable objects found in all package modules. 1567 package_dir : str 1568 Path to project directory. 1569 """ 1570 self.package_dir = os.path.join(self.sandbox, self.package_name) 1571 1572 self.files_list, self.dirs_list = get_files_dirs_list(self.package_dir) 1573 1574 self.object_cnt = 0 1575 self.docstring_cnt = 0 1576 self.docformat_cnt = 0 1577 self.doctests_count = 0 1578 self.functions = [] 1579 self.classes = [] 1580 self.methods = [] 1581 self.unittests_count = 0 1582 1583 # Parse all application files and count objects 1584 # (modules/classes/functions) and their associated docstrings. 1585 for py_file in get_files_of_type(self.files_list, 'module'): 1586 pyfile = os.path.join(self.package_dir, py_file) 1587 code = CodeParser(pyfile, self.log.debug) 1588 1589 self.object_cnt += code.object_count() 1590 self.docstring_cnt += code.docstring_count() 1591 self.docformat_cnt += code.formatted_docstrings_count 1592 self.functions += code.functions 1593 self.classes += code.classes 1594 self.methods += code.methods 1595 self.doctests_count += code.doctests_count 1596 self.unittests_count += code.unittests_count 1597 1598 # Log a bit of debugging info. 1599 self.log.debug("Found %d files: %s." % (len(self.files_list), 1600 ', '.join(self.files_list))) 1601 self.log.debug("Found %d directories: %s." % (len(self.dirs_list), 1602 ', '.join(self.dirs_list)))
1603 1604 steps['install_pkg'] = Step(['installed'])
1605 - def install_pkg(self):
1606 """Verify that package can be installed in alternate directory. 1607 1608 New attributes: 1609 installed : bool 1610 Describes whenever package has been succefully installed. 1611 """ 1612 self.sandbox_install_dir = os.path.join(self.sandbox, "tmp_install_%s" % self.package_name) 1613 1614 if self.package_type == 'egg': 1615 # Create dummy Python directories. 1616 mkdirs('%s/lib/python2.3/site-packages/' % self.sandbox_install_dir) 1617 mkdirs('%s/lib/python2.4/site-packages/' % self.sandbox_install_dir) 1618 1619 environment = {'PYTHONPATH': 1620 '%(sandbox)s/lib/python2.3/site-packages/:'\ 1621 '%(sandbox)s/lib/python2.4/site-packages/' % \ 1622 {'sandbox': self.sandbox_install_dir}, 1623 # Pass PATH to child process. 1624 'PATH': os.getenv('PATH')} 1625 rc, output = run_cmd("easy_install --no-deps --prefix %s %s" % \ 1626 (self.sandbox_install_dir, 1627 self.sandbox_pkg_file), 1628 environment) 1629 else: 1630 package_dir = os.path.join(self.sandbox, self.package_name) 1631 if not os.path.isdir(package_dir): 1632 package_dir = self.sandbox 1633 cwd = os.getcwd() 1634 os.chdir(package_dir) 1635 rc, output = run_cmd("python setup.py install --root=%s" % \ 1636 self.sandbox_install_dir) 1637 os.chdir(cwd) 1638 1639 if rc: 1640 self.log('*** Installation failed. Captured output:') 1641 # Stringify output as it may be an exception. 1642 for output_line in str(output).splitlines(): 1643 self.log(output_line) 1644 self.log('*** End of captured output.') 1645 else: 1646 self.log('Installation into %s successful.' % \ 1647 self.sandbox_install_dir) 1648 self.installed = True
1649
1650 - def compute_cheesecake_index(self):
1651 """Compute overall Cheesecake index for the package by adding up 1652 specific indexes. 1653 """ 1654 # Recursively compute all indices. 1655 max_cheesecake_index = self.index.max_value 1656 1657 # Pass Cheesecake instance to the main Index object. 1658 cheesecake_index = self.index.compute_with(self) 1659 percentage = (cheesecake_index * 100) / max_cheesecake_index 1660 1661 self.log.info("A given package can currently reach a MAXIMUM number of %d points" % max_cheesecake_index) 1662 self.log.info("Starting computation of Cheesecake index for package '%s'" % (self.package)) 1663 1664 # Print summary. 1665 if self.quiet: 1666 print "Cheesecake index: %d (%d / %d)" % (percentage, 1667 cheesecake_index, 1668 max_cheesecake_index) 1669 else: 1670 print 1671 print pad_line("=") 1672 print pad_msg("OVERALL CHEESECAKE INDEX (ABSOLUTE)", cheesecake_index) 1673 print "%s (%d out of a maximum of %d points is %d%%)" % \ 1674 (pad_msg("OVERALL CHEESECAKE INDEX (RELATIVE)", percentage), 1675 cheesecake_index, 1676 max_cheesecake_index, 1677 percentage) 1678 1679 return cheesecake_index
1680 1681 ################################################################################ 1682 ## Command line. 1683 ################################################################################ 1684
1685 -def process_cmdline_args():
1686 """Parse command-line options. 1687 """ 1688 parser = OptionParser() 1689 parser.add_option("--keep-log", action="store_true", dest="keep_log", 1690 default=False, help="don't remove log file even if run was successful") 1691 parser.add_option("--lite", action="store_true", dest="lite", 1692 default=False, help="don't run time-consuming tests (default=False)") 1693 parser.add_option("-l", "--logfile", dest="logfile", 1694 default=None, 1695 help="file to log all cheesecake messages") 1696 parser.add_option("-n", "--name", dest="name", 1697 default="", help="package name (will be retrieved via setuptools utilities, if present)") 1698 parser.add_option("-p", "--path", dest="path", 1699 default="", help="path of tar.gz/zip package on local file system") 1700 parser.add_option("-q", "--quiet", action="store_true", dest="quiet", 1701 default=False, help="only print Cheesecake index value (default=False)") 1702 parser.add_option("-s", "--sandbox", dest="sandbox", 1703 default=None, 1704 help="directory where package will be unpacked "\ 1705 "(default is to use random directory inside %s)" % tempfile.gettempdir()) 1706 parser.add_option("-t", "--static", action="store_true", dest="static", 1707 default=False, help="don't run any code from the package being tested (default=False)") 1708 parser.add_option("-u", "--url", dest="url", 1709 default="", help="package URL") 1710 parser.add_option("-v", "--verbose", action="store_true", dest="verbose", 1711 default=False, help="verbose output (default=False)") 1712 parser.add_option("-V", "--version", action="store_true", dest="version", 1713 default=False, help="Output cheesecake version and exit") 1714 1715 (options, args) = parser.parse_args() 1716 return options
1717
1718 -def main():
1719 """Display Cheesecake index for package specified via command-line options. 1720 """ 1721 options = process_cmdline_args() 1722 keep_log = options.keep_log 1723 lite = options.lite 1724 logfile = options.logfile 1725 name = options.name 1726 path = options.path 1727 quiet = options.quiet 1728 sandbox = options.sandbox 1729 static_only = options.static 1730 url = options.url 1731 verbose = options.verbose 1732 version = options.version 1733 1734 if version: 1735 print "cheesecake version %s" % VERSION 1736 sys.exit(0) 1737 1738 if not name and not url and not path: 1739 print "Error: No package name, URL or path specified (see --help)" 1740 sys.exit(1) 1741 1742 try: 1743 c = Cheesecake(name=name, url=url, path=path, sandbox=sandbox, 1744 logfile=logfile, verbose=verbose, 1745 quiet=quiet, static_only=static_only, lite=lite, 1746 keep_log=keep_log) 1747 c.compute_cheesecake_index() 1748 c.cleanup() 1749 except CheesecakeError, e: 1750 print str(e)
1751 1752 if __name__ == "__main__": 1753 main() 1754