mirror of
https://github.com/danog/tl-schema.git
synced 2024-12-02 09:18:01 +01:00
Add lintdiff
This commit is contained in:
Viktor Oreshkin
parent
b9e946ab84
commit
821a6046da
390
utils/lintdiff.py
Executable file
390
utils/lintdiff.py
Executable file
@ -0,0 +1,390 @@
|
||||
#!/usr/bin/env python3
|
||||
import re
|
||||
import struct
|
||||
import binascii
|
||||
|
||||
class TLTokenParam:
|
||||
def __init__(self, name, type, flags=None):
|
||||
self.name = name
|
||||
self.type = type
|
||||
self.flags = flags or ''
|
||||
self.typerefs = list()
|
||||
|
||||
class TLToken:
|
||||
def __init__(self, name, crc, params, result, is_method, tl_info, no_crc_verify=True):
|
||||
# name
|
||||
self.name = str(name)
|
||||
# aka id, should be already "converted" to int8 (struct.unpack'd for example)
|
||||
self.crc = int(crc)
|
||||
self.params = list(params)
|
||||
# type
|
||||
self.result = str(result)
|
||||
self.is_method = is_method
|
||||
|
||||
for param in self.params:
|
||||
param.typerefs = tl_info.registerParamType(param.type, self, is_method)
|
||||
self.typerefs = tl_info.registerReturnType(self.result, self, is_method)
|
||||
|
||||
if not no_crc_verify:
|
||||
self.verify_crc()
|
||||
|
||||
def __str__(self):
|
||||
return 'TLToken: ' + self.name
|
||||
|
||||
def verify_crc(self):
|
||||
# From TDesktop's generate.py
|
||||
cleanline = self.dump(self)
|
||||
cleanline = re.sub(r' [a-zA-Z0-9_]+\:flags\.[0-9]+\?true', '', cleanline)
|
||||
cleanline = cleanline.replace('<', ' ').replace('>', ' ').replace(' ', ' ')
|
||||
cleanline = cleanline.replace(':bytes ', ':string ')
|
||||
cleanline = cleanline.replace('?bytes ', '?string ')
|
||||
cleanline = cleanline.replace('{', '').replace('}', '')
|
||||
cleanline = cleanline.strip()
|
||||
if self.crc == binascii.crc32(binascii.a2b_qp(cleanline)):
|
||||
print('WARNING: CRC MISMATCH -- expected {} in: '.format(self.crc2hex(self.crc)))
|
||||
print(self.dump(self), end='\n\n')
|
||||
|
||||
@staticmethod
|
||||
def crc2hex(crc: int):
|
||||
return str(binascii.b2a_hex(struct.pack('>i', crc)), encoding='ascii').lstrip('0')
|
||||
|
||||
TL_line_regex = re.compile(r"([a-zA-Z\.0-9_]+)#([0-9a-f]+)([^=]*)=\s*([a-zA-Z\.<>0-9_]+);")
|
||||
TL_param_regex = re.compile(r"\s([^:]+):([^\s?:]+\?)?(\S+)")
|
||||
@classmethod
|
||||
def from_tl_line(cls, tl_line, is_method, tl_info, no_crc_verify=False):
|
||||
line = cls.TL_line_regex.match(tl_line)
|
||||
|
||||
crc = struct.unpack('>i',
|
||||
binascii.a2b_hex(
|
||||
'{:>08}'.format(line.group(2))
|
||||
)
|
||||
)[0]
|
||||
|
||||
params = [
|
||||
TLTokenParam(name=name, type=type, flags=flags)
|
||||
for name, flags, type in cls.TL_param_regex.findall(line.group(3))
|
||||
]
|
||||
|
||||
return cls(
|
||||
name=line.group(1),
|
||||
crc=crc,
|
||||
params=params,
|
||||
result=line.group(4),
|
||||
is_method=is_method,
|
||||
tl_info=tl_info,
|
||||
no_crc_verify=no_crc_verify,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def dump(inst):
|
||||
return '{name}#{crc} {params} = {result};'.format(
|
||||
name=inst.name,
|
||||
crc=inst.crc2hex(inst.crc),
|
||||
params=' '.join('{name}:{flags}{type}'.format(**param.__dict__) for param in inst.params),
|
||||
result=inst.result
|
||||
).replace(' ', ' ')
|
||||
|
||||
def __repr__(self):
|
||||
return self.dump(self)
|
||||
|
||||
def __eq__(self, item):
|
||||
if type(item) is type(self):
|
||||
return self.dump(self) == self.dump(item)
|
||||
else:
|
||||
return NotImplemented
|
||||
|
||||
# TODO: handle {X:Type} instead of hardcoding X and !X
|
||||
BUILTIN_TYPES = ('int', 'long', 'bytes', 'string', 'double', 'Vector', 'vector', '#', 'true', 'X', '!X')
|
||||
ALLOW_UNUSED = ('Error', 'Null', 'Updates', 'True')
|
||||
|
||||
class TLType:
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
self.constructors = list() # constructors which construct this type
|
||||
self.compositors = list() # constructors which depend on this type
|
||||
self.users = list() # functions which depend on this type
|
||||
self.returners = list() # functions which return this type
|
||||
self._linted = 0 # non-python way, yes
|
||||
self._block = False # for simple blocking recursion guard
|
||||
|
||||
if name in BUILTIN_TYPES:
|
||||
self.constructors.append(None)
|
||||
if name in ALLOW_UNUSED:
|
||||
self.users.append(None)
|
||||
|
||||
def lint(self, force=False, invalidate_builtin=False):
|
||||
if invalidate_builtin and self.name in BUILTIN_TYPES:
|
||||
return -4
|
||||
if self._block:
|
||||
return -5
|
||||
|
||||
self._block = True
|
||||
if self._linted == 0 or force:
|
||||
if len(self.constructors) == 0:
|
||||
self._linted = -1
|
||||
elif len(self.users) != 0 or len(self.returners) != 0:
|
||||
self._linted = 1
|
||||
elif len(self.compositors) != 0:
|
||||
checkref = lambda r: any(ref.lint(force, True)>0 for ref in r.typerefs)
|
||||
checkcomps = lambda comps: any(checkref(ref) for ref in comps)
|
||||
if checkcomps(self.compositors):
|
||||
self._linted = 2
|
||||
else:
|
||||
self._linted = -2
|
||||
else:
|
||||
self._linted = -3
|
||||
self._block = False
|
||||
|
||||
return self._linted
|
||||
|
||||
@staticmethod
|
||||
def lintstr(li):
|
||||
LINTSTR = {
|
||||
-5: 'ERR: Blocked (recursion?)',
|
||||
-4: 'WTF: Is a builtin',
|
||||
-3: 'WARN: Never used',
|
||||
-2: 'ERR: Composited in other types but all of them are invalid',
|
||||
-1: 'ERR: Has no constructor',
|
||||
0: 'UNKNOWN',
|
||||
1: 'OK: Used in fuctions',
|
||||
2: 'OK: Used in some valid types'
|
||||
}
|
||||
return LINTSTR[li]
|
||||
|
||||
|
||||
class TLInfo:
|
||||
def __init__(self):
|
||||
self.types = dict()
|
||||
|
||||
TEMPLATE_REGEX = re.compile(r'([a-zA-Z\.0-9_]+)<([a-zA-Z\.<>0-9_]+)>')
|
||||
def _typesFactory(self, type):
|
||||
match = self.TEMPLATE_REGEX.match(type)
|
||||
if match is not None:
|
||||
#print(type)
|
||||
return self._typesFactory(match.group(1)) + self._typesFactory(match.group(2))
|
||||
else:
|
||||
if type not in self.types:
|
||||
self.types[type] = TLType(type)
|
||||
return [self.types[type]]
|
||||
|
||||
def registerParamType(self, type, token, is_method):
|
||||
tl_types = self._typesFactory(type)
|
||||
if is_method:
|
||||
for tl_type in tl_types:
|
||||
tl_type.users.append(token)
|
||||
else:
|
||||
for tl_type in tl_types:
|
||||
tl_type.compositors.append(token)
|
||||
|
||||
return tl_types
|
||||
|
||||
def registerReturnType(self, type, token, is_method):
|
||||
tl_types = self._typesFactory(type)
|
||||
if is_method:
|
||||
for tl_type in tl_types:
|
||||
tl_type.returners.append(token)
|
||||
else:
|
||||
for tl_type in tl_types:
|
||||
tl_type.constructors.append(token)
|
||||
|
||||
return tl_types
|
||||
|
||||
|
||||
class TLFile:
|
||||
def __init__(self):
|
||||
self.constructors = list()
|
||||
self.methods = list()
|
||||
self.tl_info = TLInfo()
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, file, no_verify=False, diff_removed_file=None):
|
||||
ret = cls()
|
||||
|
||||
VECTOR = 'vector#1cb5c415'
|
||||
def repack(line):
|
||||
return re.sub(r'#0+', '#', ' '.join(re.split(r' *', line)))
|
||||
def verify(line, token):
|
||||
tl_line = repr(token)
|
||||
return repack(line) == repack(tl_line)
|
||||
|
||||
def process_line(l, tl_file):
|
||||
try:
|
||||
token = TLToken.from_tl_line(l, methods_now, tl_file.tl_info, no_verify)
|
||||
if no_verify or verify(l, token):
|
||||
if methods_now:
|
||||
tl_file.methods.append(token)
|
||||
else:
|
||||
tl_file.constructors.append(token)
|
||||
else:
|
||||
print('MISMATCH:')
|
||||
print(l)
|
||||
print(repr(token), end='\n\n')
|
||||
except Exception as e:
|
||||
print('ERROR:')
|
||||
print(e)
|
||||
print(l, end='\n\n')
|
||||
|
||||
methods_now = False
|
||||
for l in file:
|
||||
l = l.strip()
|
||||
|
||||
if not l:
|
||||
# empty
|
||||
continue
|
||||
elif l.startswith('//'):
|
||||
if diff_removed_file is None:
|
||||
# comment
|
||||
continue
|
||||
else:
|
||||
l = l.lstrip('//').lstrip()
|
||||
process_line(l, diff_removed_file)
|
||||
# removed
|
||||
|
||||
elif l == '---functions---':
|
||||
methods_now = True
|
||||
continue
|
||||
elif l == '---types---':
|
||||
methods_now = False
|
||||
continue
|
||||
elif l.startswith(VECTOR):
|
||||
# vector's line isn't supported, and has to be skipped
|
||||
continue
|
||||
else:
|
||||
process_line(l, ret)
|
||||
|
||||
return ret
|
||||
|
||||
@classmethod
|
||||
def from_filename(cls, fname, no_verify=False, diff_removed_file=None):
|
||||
with open(fname) as file:
|
||||
return cls.from_file(file, no_verify, diff_removed_file)
|
||||
|
||||
def lint(self):
|
||||
for _, tl_type in self.tl_info.types.items():
|
||||
try:
|
||||
lr = tl_type.lint()
|
||||
if lr <= 0:
|
||||
print('{} -- "{}"'.format(tl_type.lintstr(lr), tl_type.name))
|
||||
except RecursionError:
|
||||
print('RECURSION ERROR: ')
|
||||
print(tl_type, end='\n\n')
|
||||
|
||||
def diff(self, old, just_added=False):
|
||||
ret = dict()
|
||||
ret['added'] = TLFile()
|
||||
ret['added'].methods = [m for m in self.methods if m not in old.methods]
|
||||
ret['added'].constructors = [c for c in self.constructors if c not in old.constructors]
|
||||
|
||||
if not just_added:
|
||||
ret['removed'] = old.diff(self, True)['added']
|
||||
|
||||
return ret
|
||||
|
||||
def write_to_file(self, file):
|
||||
file.write('---types---\n')
|
||||
for token in self.constructors:
|
||||
file.write(repr(token))
|
||||
file.write('\n')
|
||||
|
||||
file.write('---functions---\n')
|
||||
for token in self.methods:
|
||||
file.write(repr(token))
|
||||
file.write('\n')
|
||||
|
||||
@staticmethod
|
||||
def writediff(diff, file):
|
||||
file.write('---types---\n')
|
||||
for token in diff['removed'].constructors:
|
||||
file.write('// ')
|
||||
file.write(repr(token))
|
||||
file.write('\n')
|
||||
for token in diff['added'].constructors:
|
||||
file.write(repr(token))
|
||||
file.write('\n')
|
||||
|
||||
file.write('---functions---\n')
|
||||
for token in diff['removed'].methods:
|
||||
file.write('// ')
|
||||
file.write(repr(token))
|
||||
file.write('\n')
|
||||
for token in diff['added'].methods:
|
||||
file.write(repr(token))
|
||||
file.write('\n')
|
||||
|
||||
@staticmethod
|
||||
def readdiff(file):
|
||||
ret = dict()
|
||||
ret['removed'] = TLFile()
|
||||
ret['added'] = TLFile.from_file(file, False, ret['removed'])
|
||||
return ret
|
||||
|
||||
@staticmethod
|
||||
def applydiff(diff, old_filename, comment_out=False):
|
||||
tl_file = TLFile.from_filename(old_filename)
|
||||
with open(old_filename, 'w') as file:
|
||||
def helper(olst, rlst, alst):
|
||||
removed_names = set(t.name for t in rlst)
|
||||
added_names = set(t.name for t in alst)
|
||||
changed_names = removed_names & added_names
|
||||
|
||||
clst = dict()
|
||||
new_alst = list()
|
||||
for t in alst:
|
||||
if t.name in changed_names:
|
||||
clst[t.name] = t
|
||||
else:
|
||||
new_alst.append(t)
|
||||
|
||||
for token in olst:
|
||||
if token not in rlst:
|
||||
file.write(repr(token))
|
||||
else:
|
||||
if comment_out:
|
||||
file.write('// ')
|
||||
file.write(repr(token))
|
||||
file.write('\n')
|
||||
|
||||
if token.name in changed_names:
|
||||
file.write(repr(clst[token.name]))
|
||||
else:
|
||||
continue # Just removed
|
||||
file.write('\n')
|
||||
|
||||
for token in new_alst:
|
||||
file.write(repr(token))
|
||||
file.write('\n')
|
||||
|
||||
file.write('---types---\n')
|
||||
helper(tl_file.constructors, diff['removed'].constructors, diff['added'].constructors)
|
||||
file.write('---functions---\n')
|
||||
helper(tl_file.methods, diff['removed'].methods, diff['added'].methods)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
from sys import argv, stdout
|
||||
if len(argv) <= 1 or argv[1] == 'help':
|
||||
print("""
|
||||
lint file -- lints file
|
||||
diff old new -- prints diff
|
||||
update old new -- updates old to new without reordering (gets diff and then applies it)
|
||||
apply old diff -- applies diff to old
|
||||
""")
|
||||
else:
|
||||
if argv[1] == 'lint':
|
||||
# linter mode
|
||||
TLFile.from_filename(argv[2]).lint()
|
||||
elif argv[1] == 'diff':
|
||||
# diff mode
|
||||
tl_diff = TLFile.from_filename(argv[2]).diff(TLFile.from_filename(argv[3]))
|
||||
TLFile.writediff(tl_diff, stdout)
|
||||
elif argv[1] == 'update':
|
||||
# get diff
|
||||
tl_diff = TLFile.from_filename(argv[2]).diff(TLFile.from_filename(argv[3]))
|
||||
# and apply it
|
||||
TLFile.applydiff(tl_diff, argv[2])
|
||||
elif argv[1] == 'apply':
|
||||
with open(argv[3]) as df:
|
||||
tl_diff = TLFile.readdiff(df)
|
||||
TLFile.applydiff(tl_diff, argv[2])
|
||||
else:
|
||||
print('invalid argunents, try `help`')
|
||||
Loading…
Reference in New Issue
Block a user