PYTHON   28

utils

Guest on 16th May 2022 05:17:14 PM

  1. """Utilities for pysource
  2. """
  3.  
  4. import types
  5. import string
  6. import compiler
  7.  
  8. # We'd better have at least *one* module in this package that demonstrates
  9. # *not* using reST for our docstrings...
  10. __docformat__ = "none"
  11.  
  12. # ----------------------------------------------------------------------
  13. PLAINTEXT = "plaintext"
  14. RESTRUCTUREDTEXT = "restructuredtext"
  15.  
  16. canonical_format = { "plaintext" : PLAINTEXT,
  17.                      "plain"     : PLAINTEXT,
  18.                      "none"      : PLAINTEXT,
  19.                      "rst"              : RESTRUCTUREDTEXT,
  20.                      "rest"             : RESTRUCTUREDTEXT,
  21.                      "rtxt"             : RESTRUCTUREDTEXT,
  22.                      "restructuredtext" : RESTRUCTUREDTEXT,
  23.                      }
  24.  
  25. def docformat(text):
  26.     """Interpret a module's __docformat__ string.
  27.  
  28.    Returns a tuple of (format,language)
  29.    """
  30.     if text == None:
  31.         return PLAINTEXT,"en"
  32.  
  33.     words = string.split(string.lower(text))
  34.  
  35.     #print words
  36.  
  37.     if len(words) == 0:
  38.         return PLAINTEXT,"en"
  39.     elif len(words) > 2:
  40.         raise ValueError,"__docformat__ may be at most two 'words'"
  41.  
  42.     if len(words) == 2:
  43.         language = string.lower(words[1])
  44.     else:
  45.         language = "en"
  46.  
  47.     try:
  48.         format = canonical_format[string.lower(words[0])]
  49.     except KeyError:
  50.         legal = canonical_format.keys()
  51.         legal.sort()
  52.         raise ValueError,"__docformat__ should be one of %s"%legal
  53.  
  54.     return format,language
  55.  
  56. # ----------------------------------------------------------------------
  57. def flatten(item):
  58.     """Retrieve some simpler representation of our AST.
  59.  
  60.    (and it's not meant to be 'theoretically' wonderful, just something
  61.    I can look at to work out how an AST works...)
  62.    """
  63.     if isinstance(item,compiler.ast.Node):
  64.         things = [item.__class__.__name__]
  65.         children = item.getChildren()
  66.         for child in children:
  67.             things.append(flatten(child))
  68.         return things
  69.     else:
  70.         return [item]
  71.  
  72. # ----------------------------------------------------------------------
  73. def treeprint(stream,item,indent=0):
  74.     """Simple pretty printer for the AST."""
  75.     if isinstance(item,compiler.ast.Node):
  76.         stream.write("\n%s<%s>"%(" "*indent,item.__class__.__name__))
  77.  
  78.         children = item.getChildren()
  79.         for child in children:
  80.             treeprint(stream,child,indent+2)
  81.  
  82.         # Fake our docstring as a sub-node (it's *really* more an attribute)
  83.         if hasattr(item,"docstring"):
  84.             stream.write("\n%s  <docstring> %s"%(" "*indent,item.docstring))
  85.  
  86.         # And ditto for a remembered assignment expression
  87.         if hasattr(item,"assign_expr"):
  88.             stream.write("\n%s  <assign_expr>"%(" "*indent))
  89.             treeprint(stream,item.assign_expr,indent+4)
  90.     else:
  91.         stream.write(" ")
  92.         stream.write(`item`)
  93.  
  94. # ----------------------------------------------------------------------
  95. def find_attr_docs(tree,verbose=0):
  96.     """Find candidates for documented attributes
  97.  
  98.    Note that after this, it may be that the AST will not garbage collect
  99.    its own nodes properly anymore, as we are adding in cross-linkages.
  100.    """
  101.  
  102.     if not isinstance(tree,compiler.ast.Node):
  103.         return
  104.  
  105.     children = tree.getChildren()
  106.  
  107.     # Might as well get our recursion done with first...
  108.     for child in children:
  109.         find_attr_docs(child,verbose)
  110.  
  111.     # I believe that only Stmt nodes can have Assign and Discard
  112.     # nodes as children
  113.     if not isinstance(tree,compiler.ast.Stmt):
  114.         return
  115.  
  116.     if len(children) == 0:
  117.         return
  118.  
  119.     pairs = []
  120.     last  = children[0]
  121.     for item in children[1:]:
  122.         pairs.append((last,item))
  123.         last = item
  124.  
  125.     for this,next in pairs:
  126.         if isinstance(this,compiler.ast.Assign) and \
  127.            isinstance(next,compiler.ast.Discard):
  128.             if verbose:
  129.                 print
  130.                 print
  131.                 print "*** Attribute docstring candidate"
  132.                 treeprint(this,4)
  133.                 treeprint(next,4)
  134.                 print
  135.  
  136.             nextexpr = next.expr
  137.             if isinstance(nextexpr,compiler.ast.Const):
  138.                 if type(nextexpr.value) == types.StringType:
  139.                     docstring = nextexpr.value
  140.                 else:
  141.                     if verbose:
  142.                         print
  143.                         print "... Discarded constant is not a string"
  144.                     continue
  145.             else:
  146.                 if verbose:
  147.                     print
  148.                     print "... Discarded expression is not a constant"
  149.                 continue
  150.  
  151.             # If there is more than one assignment attached to
  152.             # the <Assign> node, we are not interested
  153.             if len(this.nodes) > 1:
  154.                 if verbose:
  155.                     print
  156.                     print "... (but there are too many assignments in the <Assign>)"
  157.                 continue
  158.  
  159.             target = this.nodes[0]
  160.             if isinstance(target,compiler.ast.AssName):
  161.                 # Let's be cheeky and glue the docstring on...
  162.                 target.docstring = docstring
  163.             elif isinstance(target,compiler.ast.AssAttr):
  164.                 # Let's be cheeky and glue the docstring on...
  165.                 target.docstring = docstring
  166.             else:
  167.                 if verbose:
  168.                     print
  169.                     print "... (but the assignment is to a tuple/list/etc.)"
  170.                 continue
  171.  
  172.             if verbose:
  173.                 print
  174.                 print "Becomes:"
  175.                 treeprint(this,4)
  176.  
  177. # ----------------------------------------------------------------------
  178. def find_attr_vals(tree,verbose=0):
  179.     """Find attributes whose values we're interested in.
  180.  
  181.    Clearly, when this is working, it could do with being "folded" into
  182.    `find_attr_docs()`.
  183.  
  184.    Note that after this, it may be that the AST will not garbage collect
  185.    its own nodes properly anymore, as we are adding in cross-linkages.
  186.    """
  187.  
  188.     if not isinstance(tree,compiler.ast.Node):
  189.         return
  190.  
  191.     children = tree.getChildren()
  192.  
  193.     # Might as well get our recursion done with first...
  194.     for child in children:
  195.         find_attr_vals(child,verbose)
  196.  
  197.     # I believe that only Stmt nodes can have Assign and Discard
  198.     # nodes as children
  199.     if not isinstance(tree,compiler.ast.Stmt):
  200.         return
  201.  
  202.     for this in children:
  203.         if isinstance(this,compiler.ast.Assign):
  204.             if verbose:
  205.                 print
  206.                 print
  207.                 print "*** Assignment - name/value candidate"
  208.                 treeprint(this,4)
  209.                 print
  210.  
  211.             # If there is more than one assignment attached to
  212.             # the <Assign> node, we are not interested
  213.             if len(this.nodes) > 1:
  214.                 if verbose:
  215.                     print
  216.                     print "... (but there are too many assignments in the <Assign>)"
  217.                 continue
  218.  
  219.             target = this.nodes[0]
  220.             if isinstance(target,compiler.ast.AssName) or \
  221.                isinstance(target,compiler.ast.AssAttr):
  222.                 # Let's be cheeky and glue the associated expression on...
  223.                 target.assign_expr = this.expr
  224.             else:
  225.                 if verbose:
  226.                     print
  227.                     print "... (but the assignment is to a tuple/list/etc.)"
  228.                 continue
  229.  
  230.             if verbose:
  231.                 print
  232.                 print "Becomes:"
  233.                 treeprint(this,4)
  234.  
  235. # ----------------------------------------------------------------------
  236. def stringify_arg(thing):
  237.     """Return a string representation of a function argument.
  238.  
  239.    This just works for tuples of (strings or tuples (of strings ...) ...)
  240.    """
  241.     if type(thing) == types.StringType:
  242.         return thing
  243.     elif type(thing) == types.TupleType:
  244.         innards = []
  245.         for item in thing:
  246.             innards.append(stringify_arg(item))
  247.         return "(" + string.join(innards,",") + ")"
  248.     else:
  249.         raise ValueError,"Tried to stringify type %s"%type(thing)
  250.  
  251. # ----------------------------------------------------------------------
  252. def merge_args(args,defaults):
  253.     """Merge together arguments and defaults from an argument list.
  254.  
  255.    Returns a list of argument strings.
  256.    """
  257.     if args == None:
  258.         return []
  259.  
  260.     if defaults == None:
  261.         defaults = []
  262.  
  263.     # This is horrible - do it nicely later on!
  264.     argstrs = []
  265.     for item in args:
  266.         argstrs.append(stringify_arg(item))
  267.  
  268.     pos = len(args) - len(defaults)
  269.     next = 0
  270.     for index in range(pos,len(args)):
  271.         thing = defaults[next]
  272.         thing = stringify_expr(thing)
  273.         argstrs[index] = "%s=%s"%(argstrs[index],thing)
  274.         next = next + 1
  275.     return argstrs
  276.  
  277. # ----------------------------------------------------------------------
  278. def stringify_expr(thing):
  279.     """Return a very simple string representation of an expression node
  280.  
  281.    Specifically, this function aims to support stringifying things
  282.    which can be on the RHS of an assignment - is that *actually* the
  283.    same as stringifying expression nodes?
  284.    """
  285.  
  286.     # Humph - saving typing may be a good thing...
  287.     strify = stringify_expr
  288.  
  289.     #print thing.__class__.__name__
  290.    
  291.     if thing == None:
  292.         return 'None'
  293.     elif isinstance(thing,compiler.ast.Add):
  294.         return strify(thing.left) + " + " + strify(thing.right)
  295.     elif isinstance(thing,compiler.ast.And):
  296.         exprs = []
  297.         for node in thing.nodes:
  298.             exprs.append(strify(node))
  299.         return string.join(exprs," && ")
  300.     elif isinstance(thing,compiler.ast.AssAttr):
  301.         # Attribute as target of assignment
  302.         if thing.flags == compiler.consts.OP_ASSIGN:
  303.             return strify(thing.expr) + "." + thing.attrname
  304.         else:
  305.             raise ValueError,"Unexpected flag %d in %s"%(thing.flags,`thing`)
  306.     elif isinstance(thing,compiler.ast.AssName):
  307.         # Name as target of assignment, but this also means name
  308.         # as target of "in" assignment (e.g., "x in [1,2,3]"),
  309.         # which is why *we're* interested in it (since this can
  310.         # occur in list comprehensions, which can occur as the
  311.         # RHS of assignments)
  312.         if thing.flags == compiler.consts.OP_ASSIGN:
  313.             return thing.name
  314.         else:
  315.             raise ValueError,"Unexpected flag %d in %s"%(thing.flags,`thing`)
  316.     elif isinstance(thing,compiler.ast.Backquote):
  317.         return "`" + strify(thing.expr) + "`"
  318.     elif isinstance(thing,compiler.ast.Bitand):
  319.         exprs = []
  320.         for node in thing.nodes:
  321.             exprs.append(strify(node))
  322.         return string.join(exprs," & ")
  323.     elif isinstance(thing,compiler.ast.Bitor):
  324.         exprs = []
  325.         for node in thing.nodes:
  326.             exprs.append(strify(node))
  327.         return string.join(exprs," | ")
  328.     elif isinstance(thing,compiler.ast.Bitxor):
  329.         exprs = []
  330.         for node in thing.nodes:
  331.             exprs.append(strify(node))
  332.         return string.join(exprs," ^ ")
  333.     elif isinstance(thing,compiler.ast.CallFunc):
  334.         # Yuck - this is getting complicated!
  335.         # (for an example, see method `hyperlink_target` in
  336.         # restructuredtext/states.py)
  337.         str = strify(thing.node) + "("
  338.         arglist = []
  339.         if thing.args:
  340.             for arg in thing.args:
  341.                 arglist.append(strify(arg))
  342.         if thing.star_args:
  343.             arglist.append("*"+strify(thing.star_args))
  344.         if thing.dstar_args:
  345.             arglist.append("**"+strify(thing.dstar_args))
  346.         if arglist:
  347.             str += string.join(arglist,", ")
  348.         return str+")"
  349.     elif isinstance(thing,compiler.ast.Compare):
  350.         str = strify(thing.expr) + " "
  351.         for op,val in thing.ops:
  352.             str += op + " " + strify(val)
  353.         return str
  354.     elif isinstance(thing,compiler.ast.Const):
  355.         # Try not to let long strings take up too much room...
  356.         value = thing.value
  357.         if type(value) == type("") and len(value) > 50:
  358.             value = value[:47] + "..."
  359.         # Make Python decide for us if it needs quotes round it
  360.         # (and, if so, what sort)
  361.         return `value`
  362.     elif isinstance(thing,compiler.ast.Dict):
  363.         innards = []
  364.         for key,val in thing.items:
  365.             key = strify(key)
  366.             val = strify(val)
  367.             innards.append(key+":"+val)
  368.         return "{" + string.join(innards,", ") + "}"
  369.     elif isinstance(thing,compiler.ast.Div):
  370.         return strify(thing.left) + " / " + strify(thing.right)
  371.     elif isinstance(thing,compiler.ast.Ellipsis):
  372.         return "..."
  373.     elif isinstance(thing,compiler.ast.Getattr):
  374.         return strify(thing.expr) + "." + thing.attrname
  375.     elif isinstance(thing,compiler.ast.Invert):
  376.         # Bitwise negation
  377.         return "~" + strify(thing.expr)
  378.     elif isinstance(thing,compiler.ast.Keyword):
  379.         # An 'arg=value' within a function call
  380.         return thing.name + "=" + strify(thing.expr)
  381.     elif isinstance(thing,compiler.ast.Lambda):
  382.         str = "lambda "
  383.         if thing.flags != 0:
  384.             str += " <flag %d> "%thing.flags
  385.         str += string.join(merge_args(thing.argnames,thing.defaults),", ")
  386.         str += ": "
  387.         str += strify(thing.code)
  388.         return str
  389.     elif isinstance(thing,compiler.ast.LeftShift):
  390.         return strify(thing.left) + " << " + strify(thing.right)
  391.     elif isinstance(thing,compiler.ast.List):
  392.         innards = []
  393.         for item in thing.nodes:
  394.             innards.append(strify(item))
  395.         return "[" + string.join(innards,", ") + "]"
  396.     elif isinstance(thing,compiler.ast.ListComp):
  397.         str = "["+strify(thing.expr)
  398.         for node in thing.quals:
  399.             str += " "+strify(node)
  400.         return str+"]"
  401.     elif isinstance(thing,compiler.ast.ListCompFor):
  402.         str = "for "+strify(thing.assign)
  403.         str += " in "+strify(thing.list)
  404.         if thing.ifs:
  405.             for node in thing.ifs:
  406.                 str += " "+strify(node)
  407.         return str
  408.     elif isinstance(thing,compiler.ast.ListCompIf):
  409.         return "if "+strify(thing.test)
  410.     elif isinstance(thing,compiler.ast.Mod):
  411.         return strify(thing.left) + "%" + strify(thing.right)
  412.     elif isinstance(thing,compiler.ast.Mul):
  413.         return strify(thing.left) + " * " + strify(thing.right)
  414.     elif isinstance(thing,compiler.ast.Name):
  415.         return thing.name
  416.     elif isinstance(thing,compiler.ast.Not):
  417.         return "not " + strify(thing.expr)
  418.     elif isinstance(thing,compiler.ast.Or):
  419.         exprs = []
  420.         for node in thing.nodes:
  421.             exprs.append(strify(node))
  422.         return string.join(exprs," || ")
  423.     elif isinstance(thing,compiler.ast.Power):
  424.         return strify(thing.left) + " ** " + strify(thing.right)
  425.     elif isinstance(thing,compiler.ast.RightShift):
  426.         return strify(thing.left) + " >> " + strify(thing.right)
  427.     elif isinstance(thing,compiler.ast.Slice):
  428.         if thing.flags != compiler.consts.OP_APPLY:
  429.             raise ValueError,"Unexpected flag %d in %s"%(thing.flags,`thing`)
  430.         return strify(thing.expr) + "[" + \
  431.                strify(thing.lower) + ":" + strify(thing.upper) + "]"
  432.     elif isinstance(thing,compiler.ast.Sliceobj):
  433.         slicelist = []
  434.         for idx in thing.nodes:
  435.             slicelist.append(strify(idx))
  436.         return string.join(slicelist,":")
  437.     elif isinstance(thing,compiler.ast.Sub):
  438.         return strify(thing.left) + " - " + strify(thing.right)
  439.     elif isinstance(thing,compiler.ast.Subscript):
  440.         if thing.flags != compiler.consts.OP_APPLY:
  441.             raise ValueError,"Unexpected flag %d in %s"%(thing.flags,`thing`)
  442.         str = strify(thing.expr) + "["
  443.         sublist = []
  444.         for sub in thing.subs:
  445.             sublist.append(strify(sub))
  446.         return str + string.join(sublist,", ") + "]"
  447.     elif isinstance(thing,compiler.ast.Tuple):
  448.         innards = []
  449.         for item in thing.nodes:
  450.             innards.append(strify(item))
  451.         return "(" + string.join(innards,", ") + ")"
  452.     elif isinstance(thing,compiler.ast.UnaryAdd):
  453.         return "+" + strify(thing.expr)
  454.     elif isinstance(thing,compiler.ast.UnarySub):
  455.         return "-" + strify(thing.expr)
  456.     else:
  457.         return _whatsthis(thing)
  458.  
  459. def _whatsthis(thing):
  460.     # Wrong, but what else can we do?
  461.     import sys
  462.     print >>sys.stderr,"stringify_expr - don't recognise %s %s"%\
  463.           (thing.__class__.__name__,thing)
  464.     return `thing`
  465.  

Raw Paste


Login or Register to edit or fork this paste. It's free.