+++ /dev/null
-#!/usr/bin/env python
-# encoding: utf-8
-__author__ = 'David Schoonover <dsc@less.ly>'
-__date__ = '2010-11-24'
-__version__ = (0, 0, 1)
-
-__all__ = ('ResolutionError', 'Module', 'JSResolver',)
-
-
-import sys, re, jsond as json
-from itertools import chain, repeat
-from collections import defaultdict
-from subprocess import Popen, check_output, STDOUT, PIPE
-from glob import glob
-from pprint import pprint, pformat
-
-from bunch import *
-from path import path
-import pystache
-
-
-AT_REQUIRE_PAT = re.compile(r'//@require\(\s*([\'"])(.*?)\1\s*\)')
-LINE_COMMENT_PAT = re.compile(r'(.*?)(//.*?)(?:\n|$)')
-PAIR_COMMENT_PAT = re.compile(r'(.*?)(/\*.*?\*/)')
-REQUIRE_PAT = re.compile(r'\brequire\(\s*([\'"])(.*?)\1\s*\)')
-
-ROOT = path(__file__).abspath().dirname().dirname()
-LIB = ROOT/'lib/cjs'
-CWD = path('.')
-BLANK = path('')
-
-MODULE_TEMPLATE = ''
-DUMMY_TEMPLATE = ''
-DEPS_TEMPLATE = ''
-
-try:
- with (LIB/'module.js').open('rU') as f:
- MODULE_TEMPLATE = f.read()
- with (LIB/'dummy.js').open('rU') as f:
- DUMMY_TEMPLATE = f.read()
- with (LIB/'deps.js').open('rU') as f:
- DEPS_TEMPLATE = f.read()
-except Exception as ex:
- print
- print 'Error reading template file!'
- print ' ROOT={ROOT}, LIB={LIB}, module.js={}'.format(LIB/'module.js', **locals())
- raise
-
-
-class ResolutionError(Exception): pass
-
-
-def trim(s='', *chars):
- for chs in chars:
- if chs and s.endswith(chs):
- s = s[:-len(chs)]
- return s
-
-def partition(it, sep):
- """ Partitions an iterable at the first occurrence of sep, and return a 3-tuple
- containing the list of items before the separator, the separator itself,
- and after the seperator. If the separator is not found, return a 3-tuple of
- (the exhausted iterable as a list, None, []).
- """
- before = []
- it = iter(it)
- for val in it:
- if val == sep:
- break
- else:
- before.append(val)
- else:
- return (before, None, [])
- return (before, sep, list(it))
-
-
-
-def canonicalise(query, base=None):
- """ query: A module ID (relative or absolute) or filepath
- base: Optional. The referring module ID.
- """
- if isinstance(base, Module):
- base = base.id
-
- query = trim(query, '/index.cjs', '.cjs', '/index.js', '.js')
- basedir = path(base or '').dirname()
- # print "canonicalise(query={query}, basedir={basedir})".format(**locals())
-
- if query.startswith('.'):
- id = ( basedir / query ).normpath()
- else:
- id = query
-
- if id.startswith('..'):
- raise ResolutionError('Impossible to resolve {} from {}!'.format(query, base))
-
- # print " -->", str(id)
- return str(id)
-
-
-
-class Module(Bunch):
- DEFAULTS = {
- 'id' : '', # Canonical Module ID
- 'file' : None, # path
- 'name' : '', # Module Name
- 'uri' : '', # Module URI (TODO)
- 'text' : None, # Unmodified module text
- 'contents' : None, # Compiled module text
- 'outpath' : None, # Build directory
- '_requires' : None,
- '_at_requires' : None,
- }
-
-
- def __init__(self, id, file, uri=BLANK, out=None, compile=True):
- self.update(Module.DEFAULTS)
-
- self.id = id
- self.file = path(file)
- self.name = re.subn(r'\W', '', self.id.split('/').pop().capitalize())[0]
-
- if out:
- out = path(out)
- outpath = (out / self.id) + '.js'
- else:
- out = BLANK
- outpath = self.file.replace('.cjs', '.js')
- self.outpath = path(outpath)
- self.uri = str(out/uri/self.id)+'.js'
-
- self.compile()
- # print "new!", repr(self)
-
- def compile(self):
- outdir = self.outpath.dirname()
- if not outdir.exists():
- outdir.makedirs()
-
- with self.outpath.open('w') as out:
- out.write(self.contents)
-
- return self
-
- def read(self, attr='contents'):
- if self['text'] is None:
- with self.file.open('rU') as f:
- txt = f.read()
- self['text'] = txt
- template = MODULE_TEMPLATE if self.file.endswith('.cjs') else DUMMY_TEMPLATE
- self['contents'] = pystache.render(template, self)
- return self[attr]
-
- @property
- def text(self):
- return self.read('text')
-
- @property
- def contents(self):
- return self.read('contents')
-
- def findDirectives(self, text):
- for areq in AT_REQUIRE_PAT.finditer(text):
- yield canonicalise(areq.group(2), self)
-
- def findRequires(self, text):
- text = LINE_COMMENT_PAT.subn(r'\1 ', text)[0]
- text = PAIR_COMMENT_PAT.subn(r'\1 ', text)[0]
- # print " findRequires() -> %r" % line
- for mreq in REQUIRE_PAT.finditer(text):
- yield canonicalise(mreq.group(2), self)
-
- def calcRequires(self, attr):
- if self._requires is None or self._at_requires is None:
- self._at_requires = list(self.findDirectives(self.text))
- self._requires = list(self.findRequires(self.text))
- return getattr(self, attr)
-
- @property
- def atRequires(self):
- return self.calcRequires('_at_requires')
-
- @property
- def nonAtRequires(self):
- return self.calcRequires('_requires')
-
- @property
- def requires(self):
- return self.calcRequires('_at_requires') + self._requires
-
- def __json_default__(self):
- return dict( (k,self[k]) for k in 'id file name _requires _at_requires'.split(' ') )
-
- def __hash__(self):
- return hash(self.id)
-
- def __cmp__(self, other):
- return cmp(self.id, other.id)
-
- def __eq__(self, other):
- return self.id == other.id
-
- def __str__(self):
- return str(self.id)
-
- def __repr__(self):
- return '{self.__class__.__name__}(id={self.id}, file={self.file!s})'.format(**locals())
-
-
-
-
-class CommonJS(object):
- """ Compiles JS modules into browser-safe JS files. """
-
- repos = []
- modules = {} # id -> Module
- _graph = None # id -> set(dependants)
- _at_graph = None # id -> set(dependants)
-
- def __init__(self, repos, out='build', save='build', deps_name=None, clean=True, **options):
- self.modules = {}
- self.options = options
- self.deps_name = deps_name
- self.repos = set( path(path(p).abspath()+'/') for p in repos)
- self.out = None if out is '' else out
-
- if self.out is not None and clean:
- out = path(self.out)
- for f in out.glob('*'):
- if f.isdir():
- f.rmtree()
- elif f.isfile():
- f.remove()
-
- def register(self, id, file):
- mod = self.modules[id] = Module(id=id, file=file, out=self.out)
- return mod
-
- def lookup(self, query, base=None):
- absquery = path(query).abspath()
- id = canonicalise(query, base)
- # print "lookup(query={query}, base={base}) -> id={id}".format(**locals())
-
- if id in self.modules:
- return self.modules[id]
-
- for repo in self.repos:
- if absquery.startswith(repo) and absquery.isfile():
- id = canonicalise(absquery[len(repo):])
- return self.register(id, absquery)
-
- p = (repo / id).abspath()
- for f in (p+'.cjs', p/'index.cjs', p+'.js', p/'index.js'):
- if f.isfile():
- return self.register(id, f)
-
- raise ResolutionError('Unable to find file for (query={query}, id={id}, base={base}) in repos={}!'.format(map(str, self.repos), **locals()))
-
- def __iter__(self):
- for k, mod in self.modules.iteritems():
- yield k, mod
-
- def reset(self):
- self._graph = None
- self._at_graph = None
- return self
-
- def calcGraph(self, attr):
- if self._graph is None:
- graph = self._graph = defaultdict(set)
- atgraph = self._at_graph = defaultdict(set)
- for mod in self.modules.values():
- for req in mod.nonAtRequires:
- graph[req].add(mod)
- for req in mod.atRequires:
- atgraph[req].add(mod)
- return getattr(self, attr)
-
- @property
- def graph(self):
- return self.calcGraph('_graph')
-
- @property
- def atGraph(self):
- return self.calcGraph('_at_graph')
-
- def tsort(self, graph):
- p = Popen(['tsort'], stderr=sys.stderr, stdin=PIPE, stdout=PIPE)
- deps = '\n'.join( '%s %s' % (id, dep.id) for (id, deps) in graph.iteritems() for dep in deps )
- p.stdin.write(deps)
- p.stdin.close()
- if p.wait() != 0:
- raise ResolutionError('Cannot resolve dependencies! Requirements must be acyclic!')
- return p.stdout.read().strip().split('\n')
-
- @property
- def dependencies(self):
- atg = self.tsort(self.atGraph)
- nog = self.tsort(self.graph)
- atdeps = self.atGraph.keys()
-
- for dep in atg[:]:
- if dep not in atdeps:
- try: atg.remove(dep)
- except ValueError: pass
- else:
- try: nog.remove(dep)
- except ValueError: pass
-
- # print '\nAtKeys:\n', pformat(atdeps), '\n'
- # print '\nAtDeps:\n', pformat(atg), '\n'
- # print '\nNoKeys:\n', pformat(nog), '\n'
- return atg+nog
-
- def dumpDependencies(self):
- column = Popen(['column', '-s', '\t', '-t'], stderr=sys.stderr, stdin=PIPE, stdout=PIPE)
- # for mod in sorted(self.modules.values(), key=lambda m: len(m.requires), reverse=True):
- for mod in sorted(self.modules.values()):
- column.stdin.write('%s\t->\t%r\n' % (mod, sorted(mod.requires)))
- column.stdin.close()
- if column.wait() != 0:
- print >> sys.stderr, 'Some sort of error has occurred!'
- return column.stdout.read()
-
- def genDepLoader(self):
- with (path(self.out)/self.deps_name).open('w') as deps:
- deps.write( pystache.render(DEPS_TEMPLATE, uris=json.dumps(self.uris)) )
-
- @property
- def uris(self):
- return [ self.modules[d].uri for d in self.dependencies ]
-
- @property
- def scriptTags(self):
- return '\n'.join( '<script src="{}" type="text/javascript"></script>'.format(uri) for uri in self.uris )
-
- def save(self, dir):
- with (path(dir)/'.modules.json').open('w') as f:
- json.dump({
- 'modules' : self.modules,
- 'ensure' : dict( (id, list(mod.id for mod in mods)) for id,mods in self.atGraph.iteritems() ),
- 'graph' : dict( (id, list(mod.id for mod in mods)) for id,mods in self.graph.iteritems() ),
- }, f, indent=4)
- return self
-
- @staticmethod
- def discover(files, repos, **options):
- "Discover listed modules and their dependencies."
- cjs = CommonJS(repos, **options)
-
- # print >> sys.stderr, '\n[ %s ]' % f
-
- queue = [ cjs.reset().lookup(f) for f in files ]
- seen = set()
- while queue:
- mod = queue.pop(0)
- seen.add(mod)
- # print mod, "requirements:", mod.requires
- for query in mod.requires:
- # print mod, "requires", query
- req = cjs.lookup(query, mod)
- if req not in seen:
- queue.append(req)
-
- if options.get('deps_name', None):
- cjs.genDepLoader()
-
- if options.get('print_deps', None):
- print >> sys.stderr, '\n[ %s ]' % f
- print >> sys.stderr, 'All Dependencies:'
- print >> sys.stderr, cjs.dumpDependencies()
- print >> sys.stderr, ''
- print >> sys.stderr, 'Resolution:'
- print >> sys.stderr, '\n'.join(cjs.dependencies)
-
- if options.get('script_tags', None):
- print cjs.scriptTags
-
- if options.get('state', None):
- cjs.save(options['state'])
-
- return cjs
-
-
-
-
-
-
-def main():
- from optparse import OptionParser
-
- parser = OptionParser(
- usage = 'usage: %prog [options] file[...] [-- [repo_path[...]]]',
- description = 'Compiles CommonJS modules.',
- version = '%prog'+" %i.%i.%i" % __version__)
- parser.add_option("-p", "--repo-paths", default='',
- help="Comma-seperated paths to search for unqualified modules. "
- "If a path contains :, the portion afterward will be taken as the repo URL. [default: .]")
- parser.add_option("-o", "--out", default='build',
- help="Root directory to write compiled JS files. Specify '' to write to module's dir. [default: build]")
- parser.add_option("-C", "--no-clean", dest='clean', default=True, action="store_false",
- help="Do not clean the out-dir of compiled files. [default: False if -o, True otherwise]")
- parser.add_option("--print-deps", default=False, action="store_true",
- help="Prints module dependencies after compiling. [default: %default]")
- parser.add_option("-g", "--gen-deps-script", dest='deps_name', default=None,
- help="Generates a JS script with the given name which doc-writes tags for this set of modules.")
- parser.add_option("-s", "--script-tags", default=False, action="store_true",
- help="Emits script-tags for this set of modules in dependency order. [default: %default]")
- parser.add_option("-S", "--state", default='build',
- help="Directory to save build state. [default: %default]")
-
-
- (options, args) = parser.parse_args()
-
- if not args:
- parser.error("You must specify at least one JS file to resolve!")
-
- (files, sep, repos) = partition(sys.argv, '--')
- files = filter(lambda f: f in args, files)
- repos = filter(lambda f: f in args, repos)
- repos.extend( filter(None, options.repo_paths.split(',')) )
-
- # print 'files:', files, 'repos:', (repos or ['.'])
- try:
- cjs = CommonJS.discover(files=files, repos=repos or ['.'], **options.__dict__)
- except ResolutionError as ex:
- print >> sys.stderr, str(ex)
- return 1
-
- return 0
-
-if __name__ == '__main__':
- sys.exit(main())
-
projects / tanks.git / commitdiff