PYTHON 13
Aotcompile.py Guest on 3rd October 2020 02:54:31 PM
  1. # -*- python -*-
  2.  
  3. ## Copyright (C) Free Software Foundation
  4. ## Written by Gary Benson <[email protected]>
  5. ##
  6. ## This program is free software; you can redistribute it and/or modify
  7. ## it under the terms of the GNU General Public License as published by
  8. ## the Free Software Foundation; either version 2 of the License, or
  9. ## (at your option) any later version.
  10. ##
  11. ## This program is distributed in the hope that it will be useful,
  12. ## but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14. ## GNU General Public License for more details.
  15.  
  16. import classfile
  17. import copy
  18. # The md5 module is deprecated in Python 2.5
  19. try:
  20.     from hashlib import md5
  21. except ImportError:
  22.     from md5 import md5
  23. import operator
  24. import os
  25. import sys
  26. import cStringIO as StringIO
  27. import zipfile
  28.  
  29. PATHS = {"make":   "/usr/bin/make",
  30.          "gcj":    "/usr/bin/gcj",
  31.          "dbtool": "/usr/bin/gcj-dbtool"}
  32.  
  33. MAKEFLAGS = []
  34. GCJFLAGS = ["-fPIC", "-findirect-dispatch", "-fjni"]
  35. LDFLAGS = ["-Wl,-Bsymbolic"]
  36.  
  37. MAX_CLASSES_PER_JAR = 1024
  38. MAX_BYTES_PER_JAR = 1048576
  39.  
  40. MAKEFILE = "Makefile"
  41.  
  42. MAKEFILE_HEADER = '''\
  43. GCJ = %(gcj)s
  44. DBTOOL = %(dbtool)s
  45. GCJFLAGS = %(gcjflags)s
  46. LDFLAGS = %(ldflags)s
  47.  
  48. %%.o: %%.jar
  49.         $(GCJ) -c $(GCJFLAGS) $< -o [email protected]
  50.  
  51. TARGETS = \\
  52. %(targets)s
  53.  
  54. all: $(TARGETS)'''
  55.  
  56. MAKEFILE_JOB = '''
  57. %(base)s_SOURCES = \\
  58. %(jars)s
  59.  
  60. %(base)s_OBJECTS = \\
  61. $(%(base)s_SOURCES:.jar=.o)
  62.  
  63. %(dso)s: $(%(base)s_OBJECTS)
  64.         $(GCJ) -shared $(GCJFLAGS) $(LDFLAGS) $^ -o [email protected]
  65.  
  66. %(db)s: $(%(base)s_SOURCES)
  67.         $(DBTOOL) -n [email protected] 64
  68.         for jar in $^; do \\
  69.            $(DBTOOL) -f [email protected] $$jar \\
  70.                %(libdir)s/%(dso)s; \\
  71.        done'''
  72.  
  73. ZIPMAGIC, CLASSMAGIC = "PK\x03\x04", "\xca\xfe\xba\xbe"
  74.  
  75. class Error(Exception):
  76.     pass
  77.  
  78. class Compiler:
  79.     def __init__(self, srcdir, libdir, prefix = None):
  80.         self.srcdir = os.path.abspath(srcdir)
  81.         self.libdir = os.path.abspath(libdir)
  82.         if prefix is None:
  83.             self.dstdir = self.libdir
  84.         else:
  85.             self.dstdir = os.path.join(prefix, self.libdir.lstrip(os.sep))
  86.  
  87.         # Calling code may modify these parameters
  88.         self.gcjflags = copy.copy(GCJFLAGS)
  89.         self.ldflags = copy.copy(LDFLAGS)
  90.         self.makeflags = copy.copy(MAKEFLAGS)
  91.         self.exclusions = []
  92.  
  93.     def compile(self):
  94.         """Search srcdir for classes and jarfiles, then generate
  95.        solibs and mappings databases for them all in libdir."""
  96.         if not os.path.isdir(self.dstdir):
  97.             os.makedirs(self.dstdir)
  98.         oldcwd = os.getcwd()
  99.         os.chdir(self.dstdir)
  100.         try:            
  101.             jobs = self.getJobList()
  102.             if not jobs:
  103.                 raise Error, "nothing to do"
  104.             self.writeMakefile(MAKEFILE, jobs)
  105.             for job in jobs:
  106.                 job.writeJars()
  107.             system([PATHS["make"]] + self.makeflags)
  108.             for job in jobs:
  109.                 job.clean()
  110.             os.unlink(MAKEFILE)
  111.         finally:
  112.             os.chdir(oldcwd)
  113.  
  114.     def getJobList(self):
  115.         """Return all jarfiles and class collections in srcdir."""
  116.         jobs = weed_jobs(find_jobs(self.srcdir, self.exclusions))
  117.         set_basenames(jobs)
  118.         return jobs
  119.  
  120.     def writeMakefile(self, path, jobs):
  121.         """Generate a makefile to build the solibs and mappings
  122.        databases for the specified list of jobs."""
  123.         fp = open(path, "w")
  124.         print >>fp, MAKEFILE_HEADER % {
  125.             "gcj": PATHS["gcj"],
  126.             "dbtool": PATHS["dbtool"],
  127.             "gcjflags": " ".join(self.gcjflags),
  128.             "ldflags": " ".join(self.ldflags),
  129.             "targets": " \\\n".join(reduce(operator.add, [
  130.                 (job.dsoName(), job.dbName()) for job in jobs]))}
  131.         for job in jobs:
  132.             values = job.ruleArguments()
  133.             values["libdir"] = self.libdir
  134.             print >>fp, MAKEFILE_JOB % values
  135.         fp.close()
  136.  
  137. def find_jobs(dir, exclusions = ()):
  138.     """Scan a directory and find things to compile: jarfiles (zips,
  139.    wars, ears, rars, etc: we go by magic rather than file extension)
  140.    and directories of classes."""
  141.     def visit((classes, zips), dir, items):
  142.         for item in items:
  143.             path = os.path.join(dir, item)
  144.             if os.path.islink(path) or not os.path.isfile(path):
  145.                 continue
  146.             magic = open(path, "r").read(4)
  147.             if magic == ZIPMAGIC:
  148.                 zips.append(path)
  149.             elif magic == CLASSMAGIC:
  150.                 classes.append(path)
  151.     classes, paths = [], []
  152.     os.path.walk(dir, visit, (classes, paths))
  153.     # Convert the list of classes into a list of directories
  154.     while classes:
  155.         # XXX this requires the class to be correctly located in its heirachy.
  156.         path = classes[0][:-len(os.sep + classname(classes[0]) + ".class")]
  157.         paths.append(path)
  158.         classes = [cls for cls in classes if not cls.startswith(path)]
  159.     # Handle exclusions.  We're really strict about them because the
  160.     # option is temporary in aot-compile-rpm and dead options left in
  161.     # specfiles will hinder its removal.
  162.     for path in exclusions:
  163.         if path in paths:
  164.             paths.remove(path)
  165.         else:
  166.             raise Error, "%s: path does not exist or is not a job" % path
  167.     # Build the list of jobs
  168.     jobs = []
  169.     paths.sort()
  170.     for path in paths:
  171.         if os.path.isfile(path):
  172.             job = JarJob(path)
  173.         else:
  174.             job = DirJob(path)
  175.         if len(job.classes):
  176.             jobs.append(job)
  177.     return jobs
  178.  
  179. class Job:
  180.     """A collection of classes that will be compiled as a unit."""
  181.    
  182.     def __init__(self, path):
  183.         self.path, self.classes, self.blocks = path, {}, None
  184.         self.classnames = {}
  185.  
  186.     def addClass(self, bytes, name):
  187.         """Subclasses call this from their __init__ method for
  188.        every class they find."""
  189.         digest = md5(bytes).digest()
  190.         self.classes[digest] = bytes
  191.         self.classnames[digest] = name
  192.  
  193.     def __makeBlocks(self):
  194.         """Split self.classes into chunks that can be compiled to
  195.        native code by gcj.  In the majority of cases this is not
  196.        necessary -- the job will have come from a jarfile which will
  197.        be equivalent to the one we generate -- but this only happens
  198.        _if_ the job was a jarfile and _if_ the jarfile isn't too big
  199.        and _if_ the jarfile has the correct extension and _if_ all
  200.        classes are correctly named and _if_ the jarfile has no
  201.        embedded jarfiles.  Fitting a special case around all these
  202.        conditions is tricky to say the least.
  203.  
  204.        Note that this could be called at the end of each subclass's
  205.        __init__ method.  The reason this is not done is because we
  206.        need to parse every class file.  This is slow, and unnecessary
  207.        if the job is subsetted."""
  208.         names = {}
  209.         for hash, bytes in self.classes.items():
  210.             try:
  211.                 name = classname(bytes)
  212.             except:
  213.                 warn("job %s: class %s malformed or not a valid class file" \
  214.                      % (self.path, self.classnames[hash]))
  215.                 raise
  216.             if not names.has_key(name):
  217.                 names[name] = []
  218.             names[name].append(hash)
  219.         names = names.items()
  220.         # We have to sort somehow, or the jars we generate
  221.         # We sort by name in a simplistic attempt to keep related
  222.         # classes together so inter-class optimisation can happen.
  223.         names.sort()
  224.         self.blocks, bytes = [[]], 0
  225.         for name, hashes in names:
  226.             for hash in hashes:
  227.                 if len(self.blocks[-1]) >= MAX_CLASSES_PER_JAR \
  228.                    or bytes >= MAX_BYTES_PER_JAR:
  229.                     self.blocks.append([])
  230.                     bytes = 0
  231.                 self.blocks[-1].append((name, hash))
  232.                 bytes += len(self.classes[hash])
  233.  
  234.     # From Archit Shah:
  235.     #   The implementation and the documentation don't seem to match.
  236.     #  
  237.     #    [a, b].isSubsetOf([a]) => True
  238.     #  
  239.     #   Identical copies of all classes this collection do not exist
  240.     #   in the other. I think the method should be named isSupersetOf
  241.     #   and the documentation should swap uses of "this" and "other"
  242.     #
  243.     # XXX think about this when I've had more sleep...
  244.     def isSubsetOf(self, other):
  245.         """Returns True if identical copies of all classes in this
  246.        collection exist in the other."""
  247.         for item in other.classes.keys():
  248.             if not self.classes.has_key(item):
  249.                 return False
  250.         return True
  251.  
  252.     def __targetName(self, ext):
  253.         return self.basename + ext
  254.  
  255.     def tempJarName(self, num):
  256.         return self.__targetName(".%d.jar" % (num + 1))
  257.  
  258.     def tempObjName(self, num):
  259.         return self.__targetName(".%d.o" % (num + 1))
  260.  
  261.     def dsoName(self):
  262.         """Return the filename of the shared library that will be
  263.        built from this job."""
  264.         return self.__targetName(".so")
  265.  
  266.     def dbName(self):
  267.         """Return the filename of the mapping database that will be
  268.        built from this job."""
  269.         return self.__targetName(".db")
  270.  
  271.     def ruleArguments(self):
  272.         """Return a dictionary of values that when substituted
  273.        into MAKEFILE_JOB will create the rules required to build
  274.        the shared library and mapping database for this job."""
  275.         if self.blocks is None:
  276.             self.__makeBlocks()
  277.         return {
  278.             "base": "".join(
  279.                 [c.isalnum() and c or "_" for c in self.dsoName()]),
  280.             "jars": " \\\n".join(
  281.                 [self.tempJarName(i) for i in xrange(len(self.blocks))]),
  282.             "dso": self.dsoName(),
  283.             "db": self.dbName()}
  284.  
  285.     def writeJars(self):
  286.         """Generate jarfiles that can be native compiled by gcj."""
  287.         if self.blocks is None:
  288.             self.__makeBlocks()
  289.         for block, i in zip(self.blocks, xrange(len(self.blocks))):
  290.             jar = zipfile.ZipFile(self.tempJarName(i), "w", zipfile.ZIP_STORED)
  291.             for name, hash in block:
  292.                 jar.writestr(
  293.                     zipfile.ZipInfo("%s.class" % name), self.classes[hash])
  294.             jar.close()
  295.  
  296.     def clean(self):
  297.         """Delete all temporary files created during this job's build."""
  298.         if self.blocks is None:
  299.             self.__makeBlocks()
  300.         for i in xrange(len(self.blocks)):
  301.             os.unlink(self.tempJarName(i))
  302.             os.unlink(self.tempObjName(i))
  303.  
  304. class JarJob(Job):
  305.     """A Job whose origin was a jarfile."""
  306.  
  307.     def __init__(self, path):
  308.         Job.__init__(self, path)
  309.         self._walk(zipfile.ZipFile(path, "r"))
  310.  
  311.     def _walk(self, zf):
  312.         for name in zf.namelist():
  313.             bytes = zf.read(name)
  314.             if bytes.startswith(ZIPMAGIC):
  315.                 self._walk(zipfile.ZipFile(StringIO.StringIO(bytes)))
  316.             elif bytes.startswith(CLASSMAGIC):
  317.                 self.addClass(bytes, name)
  318.  
  319. class DirJob(Job):
  320.     """A Job whose origin was a directory of classfiles."""
  321.  
  322.     def __init__(self, path):
  323.         Job.__init__(self, path)
  324.         os.path.walk(path, DirJob._visit, self)
  325.  
  326.     def _visit(self, dir, items):
  327.         for item in items:
  328.             path = os.path.join(dir, item)
  329.             if os.path.islink(path) or not os.path.isfile(path):
  330.                 continue
  331.             fp = open(path, "r")
  332.             magic = fp.read(4)
  333.             if magic == CLASSMAGIC:
  334.                 self.addClass(magic + fp.read(), name)
  335.    
  336. def weed_jobs(jobs):
  337.     """Remove any jarfiles that are completely contained within
  338.    another.  This is more common than you'd think, and we only
  339.    need one nativified copy of each class after all."""
  340.     jobs = copy.copy(jobs)
  341.     while True:
  342.         for job1 in jobs:
  343.             for job2 in jobs:
  344.                 if job1 is job2:
  345.                     continue
  346.                 if job1.isSubsetOf(job2):
  347.                     msg = "subsetted %s" % job2.path
  348.                     if job2.isSubsetOf(job1):
  349.                         if (isinstance(job1, DirJob) and
  350.                             isinstance(job2, JarJob)):
  351.                             # In the braindead case where a package
  352.                             # contains an expanded copy of a jarfile
  353.                             # the jarfile takes precedence.
  354.                             continue
  355.                         msg += " (identical)"
  356.                     warn(msg)
  357.                     jobs.remove(job2)
  358.                     break
  359.             else:
  360.                 continue
  361.             break
  362.         else:
  363.             break
  364.         continue
  365.     return jobs
  366.  
  367. def set_basenames(jobs):
  368.     """Ensure that each jarfile has a different basename."""
  369.     names = {}
  370.     for job in jobs:
  371.         name = os.path.basename(job.path)
  372.         if not names.has_key(name):
  373.             names[name] = []
  374.         names[name].append(job)
  375.     for name, set in names.items():
  376.         if len(set) == 1:
  377.             set[0].basename = name
  378.             continue
  379.         # prefix the jar filenames to make them unique
  380.         # XXX will not work in most cases -- needs generalising
  381.         set = [(job.path.split(os.sep), job) for job in set]
  382.         minlen = min([len(bits) for bits, job in set])
  383.         set = [(bits[-minlen:], job) for bits, job in set]
  384.         bits = apply(zip, [bits for bits, job in set])
  385.         while True:
  386.             row = bits[-2]
  387.             for bit in row[1:]:
  388.                 if bit != row[0]:
  389.                     break
  390.             else:
  391.                 del bits[-2]
  392.                 continue
  393.             break
  394.         set = zip(
  395.             ["_".join(name) for name in apply(zip, bits[-2:])],
  396.             [job for bits, job in set])
  397.         for name, job in set:
  398.             warn("building %s as %s" % (job.path, name))
  399.             job.basename = name
  400.     # XXX keep this check until we're properly general
  401.     names = {}
  402.     for job in jobs:
  403.         name = job.basename
  404.         if names.has_key(name):
  405.             raise Error, "%s: duplicate jobname" % name
  406.         names[name] = 1
  407.  
  408. def system(command):
  409.     """Execute a command."""
  410.     status = os.spawnv(os.P_WAIT, command[0], command)
  411.     if status > 0:
  412.         raise Error, "%s exited with code %d" % (command[0], status)
  413.     elif status < 0:
  414.         raise Error, "%s killed by signal %d" % (command[0], -status)
  415.  
  416. def warn(msg):
  417.     """Print a warning message."""
  418.     print >>sys.stderr, "%s: warning: %s" % (
  419.         os.path.basename(sys.argv[0]), msg)
  420.  
  421. def classname(bytes):
  422.     """Extract the class name from the bytes of a class file."""
  423.     klass = classfile.Class(bytes)
  424.     return klass.constants[klass.constants[klass.name][1]][1]

Paste is for source code and general debugging text.

Login or Register to edit, delete and keep track of your pastes and more.

Raw Paste

Login or Register to edit or fork this paste. It's free.