def default(self, o):

if hasattr(o, '__dict__'):

d = o.__dict__

# workaround: decode strings if it's not Python3 code

if not is_python3:

for k in d:

if isinstance(d[k], str):

if k == 's':

d[k] = lines[d['start']:d['end']]

else:

d[k] = d[k].decode(enc)

d['type'] = o.__class__.__name__

return d

else:

try:

if is_python3:

encoder = AstEncoder()

else:

encoder = AstEncoder(encoding=enc)

tree = parse_file(filename)

encoded = encoder.encode(tree)

f = open(output, "w")

f.write(encoded)

f.close()

finally:

# write marker file to signal write end

f = open(end_mark, "w")

#print 'parse_json'

try:

if is_python3:

encoder = AstEncoder()

else:

encoder = AstEncoder(encoding=enc)

tree = parse_file(filename)

encoded = encoder.encode(tree)

return encoded

except:

global enc, lines

enc, enc_len = detect_encoding(filename)

f = codecs.open(filename, 'r', enc)

lines = f.read()

# remove BOM

lines = re.sub(u'\ufeff', ' ', lines)

# replace the encoding decl by spaces to fool python parser

# otherwise you get 'encoding decl in unicode string' syntax error

# print('enc:', enc, 'enc_len', enc_len)

if enc_len > 0:

lines = re.sub('#.*coding\s*[:=]\s*[\w\d\-]+', '#' + ' ' * (enc_len-1), lines)

f.close()

tree = ast.parse(string)

improve_ast(tree, string)

if filename:

tree.filename = filename

fin = open(path, 'rb')

prefix = str(fin.read(80))

encs = re.findall('#.*coding\s*[:=]\s*([\w\d\-]+)', prefix)

decl = re.findall('#.*coding\s*[:=]\s*[\w\d\-]+', prefix)

if encs:

enc1 = encs[0]

enc_len = len(decl[0])

try:

info = codecs.lookup(enc1)

# print('lookedup: ', info)

except LookupError:

# print('encoding not exist: ' + enc1)

return 'latin1', enc_len

return enc1, enc_len

else:

global line_starts

idx = 0

line_starts = [0]

while idx < len(s):

if s[idx] == '\n':

line_starts.append(idx + 1)

line = 0

for start in line_starts:

if idx < start:

break

line += 1

col = idx - line_starts[line - 1]

if isinstance(node, list):

for n in node:

improve_node(n, s)

elif isinstance(node, AST):

find_start(node, s)

find_end(node, s)

add_missing_names(node, s)

for f in node_fields(node):

ret = None # default value

if hasattr(node, 'start'):

ret = node.start

elif isinstance(node, list):

if node != []:

ret = find_start(node[0], s)

elif isinstance(node, Module):

if node.body != []:

ret = find_start(node.body[0], s)

elif isinstance(node, BinOp):

leftstart = find_start(node.left, s)

if leftstart != None:

ret = leftstart

else:

ret = map_idx(node.lineno, node.col_offset)

elif hasattr(node, 'lineno'):

if node.col_offset >= 0:

ret = map_idx(node.lineno, node.col_offset)

else: # special case for """ strings

i = map_idx(node.lineno, node.col_offset)

while i > 0 and i + 2 < len(s) and s[i:i + 3] != '"""' and s[i:i + 3] != "'''":

i -= 1

ret = i

else:

return None

if ret == None and hasattr(node, 'lineno'):

raise TypeError("got None for node that has lineno", node)

if isinstance(node, AST) and ret != None:

node.start = ret

the_end = None

if hasattr(node, 'end'):

return node.end

elif isinstance(node, list):

if node != []:

the_end = find_end(node[-1], s)

elif isinstance(node, Module):

if node.body != []:

the_end = find_end(node.body[-1], s)

elif isinstance(node, Expr):

the_end = find_end(node.value, s)

elif isinstance(node, Str):

i = find_start(node, s)

while s[i] != '"' and s[i] != "'":

i += 1

if i + 2 < len(s) and s[i:i + 3] == '"""':

q = '"""'

i += 3

elif i + 2 < len(s) and s[i:i + 3] == "'''":

q = "'''"

i += 3

elif s[i] == '"':

q = '"'

i += 1

elif s[i] == "'":

q = "'"

i += 1

else:

print("illegal quote:", i, s[i])

q = ''

if q != '':

the_end = end_seq(s, q, i)

elif isinstance(node, Name):

the_end = find_start(node, s) + len(node.id)

elif isinstance(node, Attribute):

the_end = end_seq(s, node.attr, find_end(node.value, s))

elif isinstance(node, FunctionDef):

the_end = find_end(node.body, s)

elif isinstance(node, Lambda):

the_end = find_end(node.body, s)

elif isinstance(node, ClassDef):

the_end = find_end(node.body, s)

# print will be a Call in Python 3

elif not is_python3 and isinstance(node, Print):

the_end = start_seq(s, '\n', find_start(node, s))

elif isinstance(node, Call):

start = find_end(node.func, s)

if start != None:

the_end = match_paren(s, '(', ')', start)

elif isinstance(node, Yield):

the_end = find_end(node.value, s)

elif isinstance(node, Return):

if node.value != None:

the_end = find_end(node.value, s)

else:

the_end = find_start(node, s) + len('return')

elif (isinstance(node, For) or

isinstance(node, While) or

isinstance(node, If) or

isinstance(node, IfExp)):

if node.orelse != []:

the_end = find_end(node.orelse, s)

else:

the_end = find_end(node.body, s)

elif isinstance(node, Assign) or isinstance(node, AugAssign):

the_end = find_end(node.value, s)

elif isinstance(node, BinOp):

the_end = find_end(node.right, s)

elif isinstance(node, BoolOp):

the_end = find_end(node.values[-1], s)

elif isinstance(node, Compare):

the_end = find_end(node.comparators[-1], s)

elif isinstance(node, UnaryOp):

the_end = find_end(node.operand, s)

elif isinstance(node, Num):

the_end = find_start(node, s) + len(str(node.n))

elif isinstance(node, List):

the_end = match_paren(s, '[', ']', find_start(node, s));

elif isinstance(node, Subscript):

the_end = match_paren(s, '[', ']', find_start(node, s));

elif isinstance(node, Tuple):

if node.elts != []:

the_end = find_end(node.elts[-1], s)

elif isinstance(node, Dict):

the_end = match_paren(s, '{', '}', find_start(node, s));

elif ((not is_python3 and isinstance(node, TryExcept)) or

(is_python3 and isinstance(node, Try))):

if node.orelse != []:

the_end = find_end(node.orelse, s)

elif node.handlers != []:

the_end = find_end(node.handlers, s)

else:

the_end = find_end(node.body, s)

elif isinstance(node, ExceptHandler):

the_end = find_end(node.body, s)

elif isinstance(node, Pass):

the_end = find_start(node, s) + len('pass')

elif isinstance(node, Break):

the_end = find_start(node, s) + len('break')

elif isinstance(node, Continue):

the_end = find_start(node, s) + len('continue')

elif isinstance(node, Global):

the_end = start_seq(s, '\n', find_start(node, s))

elif isinstance(node, Import):

the_end = find_start(node, s) + len('import')

elif isinstance(node, ImportFrom):

the_end = find_start(node, s) + len('from')

else: # can't determine node end, set to 3 chars after start

start = find_start(node, s)

if start != None:

the_end = start + 3

if isinstance(node, AST) and the_end != None:

node.end = the_end

if hasattr(node, 'extra_attr'):

return

if isinstance(node, list):

for n in node:

add_missing_names(n, s)

elif isinstance(node, ClassDef):

head = find_start(node, s)

start = s.find("class", head) + len("class")

if start != None:

node.name_node = str_to_name(s, start)

node._fields += ('name_node',)

elif isinstance(node, FunctionDef):

# skip to "def" because it may contain decorators like @property

head = find_start(node, s)

start = s.find("def", head) + len("def")

if start != None:

node.name_node = str_to_name(s, start)

node._fields += ('name_node',)

# keyword_start = find_start(node, s)

# node.keyword_node = str_to_name(s, keyword_start)

# node._fields += ('keyword_node',)

if node.args.vararg != None:

if len(node.args.args) > 0:

vstart = find_end(node.args.args[-1], s)

else:

vstart = find_end(node.name_node, s)

if vstart != None:

vname = str_to_name(s, vstart)

node.vararg_name = vname

else:

node.vararg_name = None

node._fields += ('vararg_name',)

if node.args.kwarg != None:

if len(node.args.args) > 0:

kstart = find_end(node.args.args[-1], s)

else:

kstart = find_end(node.vararg_name, s)

if kstart:

kname = str_to_name(s, kstart)

node.kwarg_name = kname

else:

node.kwarg_name = None

node._fields += ('kwarg_name',)

elif isinstance(node, Attribute):

start = find_end(node.value, s)

if start is not None:

name = str_to_name(s, start)

node.attr_name = name

node._fields = ('value', 'attr_name') # remove attr for node size accuracy

elif isinstance(node, Compare):

start = find_start(node, s)

if start is not None:

node.opsName = convert_ops(node.ops, s, start)

node._fields += ('opsName',)

elif (isinstance(node, BoolOp) or

isinstance(node, BinOp) or

isinstance(node, UnaryOp) or

isinstance(node, AugAssign)):

if hasattr(node, 'left'):

start = find_end(node.left, s)

else:

start = find_start(node, s)

if start is not None:

ops = convert_ops([node.op], s, start)

else:

ops = []

if ops != []:

node.op_node = ops[0]

node._fields += ('op_node',)

elif isinstance(node, Num):

if isinstance(node.n, int) or (not is_python3 and isinstance(node.n, long)):

type = 'int'

node.n = str(node.n)

elif isinstance(node.n, float):

type = 'float'

node.n = str(node.n)

elif isinstance(node.n, complex):

type = 'complex'

node.real = node.n.real

node.imag = node.n.imag

node._fields += ('real', 'imag')

node.num_type = type

node._fields += ('num_type',)

try:

return s.index(pat, start)

except ValueError:

try:

return s.index(pat, start) + len(pat)

except ValueError:

while start < len(s) and s[start] != open:

start += 1

if start >= len(s):

return len(s)

left = 1

i = start + 1

while left > 0 and i < len(s):

if s[i] == open:

left += 1

elif s[i] == close:

left -= 1

i += 1

i = start;

while i < len(s) and not is_alpha(s[i]):

i += 1

name_start = i

ret = []

while i < len(s) and is_alpha(s[i]):

ret.append(s[i])

i += 1

name_end = i

id1 = ''.join(ret)

if id1 == '':

return None

else:

name = Name(id1, None)

name.start = name_start

name.end = name_end

name.lineno, name.col_offset = map_line_col(name_start)

syms = []

for op in ops:

if type(op) in ops_map:

syms.append(ops_map[type(op)])

else:

print("[WARNING] operator %s is missing from ops_map, "

"please report the bug on GitHub" % op)

i = start

j = 0

ret = []

while i < len(s) and j < len(syms):

oplen = len(syms[j])

if s[i:i + oplen] == syms[j]:

op_node = Name(syms[j], None)

op_node.start = i

op_node.end = i + oplen

op_node.lineno, op_node.col_offset = map_line_col(i)

ret.append(op_node)

j += 1

i = op_node.end

else:

i += 1

# compare:

Eq: '==',

NotEq: '!=',

LtE: '<=',

Lt: '<',

GtE: '>=',

Gt: '>',

NotIn: 'not in',

In: 'in',

IsNot: 'is not',

Is: 'is',

# BoolOp

Or: 'or',

And: 'and',

Not: 'not',

Invert: '~',

# bit operators

BitOr: '|',

BitAnd: '&',

BitXor: '^',

RShift: '>>',

LShift: '<<',

# BinOp

Add: '+',

Sub: '-',

Mult: '*',

Div: '/',

FloorDiv: '//',

Mod: '%',

Pow: '**',

# UnaryOp

USub: '-',

UAdd: '+',

ret = []

for field in node._fields:

if field != 'ctx' and hasattr(node, field):

ret.append(getattr(node, field))

if hasattr(node, 'node_source'):

return node.node_source

else:

if hasattr(node, 'start'):

return node.start

else:

if hasattr(node, 'end'):

return node.end

else:

return (c == '_'

or ('0' <= c <= '9')

or ('a' <= c <= 'z')