From d0a263cdd019116565682896d115ecd662515f78 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:16 -0400 Subject: [PATCH 01/25] qapi/expr: Comment cleanup The linter yaps after 0825f62c842. Fix this trivial issue to restore the linter baseline. (Yes, ideally -- and soon -- the linter will be part of CI so we don't clutter up the log with fixups. For now, though, the baseline is useful for testing intermediate commits as types are added to the QAPI library.) Signed-off-by: John Snow Message-Id: <20210421182032.3521476-2-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index 540b3982b1..c207481f7e 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -241,7 +241,7 @@ def check_enum(expr, info): source = "%s '%s'" % (source, member_name) # Enum members may start with a digit if member_name[0].isdigit(): - member_name = 'd' + member_name # Hack: hide the digit + member_name = 'd' + member_name # Hack: hide the digit check_name_lower(member_name, info, source, permit_upper=permissive, permit_underscore=permissive) From b7341b89c9c0212fef7b38b04ffaf39ea73bfca9 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:17 -0400 Subject: [PATCH 02/25] qapi/expr.py: Remove 'info' argument from nested check_if_str The function can just use the argument from the scope above. Otherwise, we get shadowed argument errors because the parameter name clashes with the name of a variable already in-scope. Signed-off-by: John Snow Reviewed-by: Eduardo Habkost Reviewed-by: Cleber Rosa Message-Id: <20210421182032.3521476-3-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index c207481f7e..3fda5d5082 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -122,7 +122,7 @@ def check_flags(expr, info): def check_if(expr, info, source): - def check_if_str(ifcond, info): + def check_if_str(ifcond): if not isinstance(ifcond, str): raise QAPISemError( info, @@ -142,9 +142,9 @@ def check_if(expr, info, source): raise QAPISemError( info, "'if' condition [] of %s is useless" % source) for elt in ifcond: - check_if_str(elt, info) + check_if_str(elt) else: - check_if_str(ifcond, info) + check_if_str(ifcond) expr['if'] = [ifcond] From 0f231dcf2921fa8bc475d222a8ef81e67d4019e8 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:18 -0400 Subject: [PATCH 03/25] qapi/expr.py: Check for dict instead of OrderedDict OrderedDict is a subtype of dict, so we can check for a more general form. These functions do not themselves depend on it being any particular type. Signed-off-by: John Snow Reviewed-by: Eduardo Habkost Reviewed-by: Cleber Rosa Message-Id: <20210421182032.3521476-4-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index 3fda5d5082..b4bbcd54c0 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -14,7 +14,6 @@ # This work is licensed under the terms of the GNU GPL, version 2. # See the COPYING file in the top-level directory. -from collections import OrderedDict import re from .common import c_name @@ -149,7 +148,7 @@ def check_if(expr, info, source): def normalize_members(members): - if isinstance(members, OrderedDict): + if isinstance(members, dict): for key, arg in members.items(): if isinstance(arg, dict): continue @@ -180,7 +179,7 @@ def check_type(value, info, source, if not allow_dict: raise QAPISemError(info, "%s should be a type name" % source) - if not isinstance(value, OrderedDict): + if not isinstance(value, dict): raise QAPISemError(info, "%s should be an object or type name" % source) From 59b5556ce8c1d3dc1f2c6445ca32f2e515114a8e Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:19 -0400 Subject: [PATCH 04/25] qapi/expr.py: constrain incoming expression types mypy does not know the types of values stored in Dicts that masquerade as objects. Help the type checker out by constraining the type. Signed-off-by: John Snow Message-Id: <20210421182032.3521476-5-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index b4bbcd54c0..06a0081001 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -15,9 +15,20 @@ # See the COPYING file in the top-level directory. import re +from typing import Dict, Optional from .common import c_name from .error import QAPISemError +from .parser import QAPIDoc +from .source import QAPISourceInfo + + +# Deserialized JSON objects as returned by the parser. +# The values of this mapping are not necessary to exhaustively type +# here (and also not practical as long as mypy lacks recursive +# types), because the purpose of this module is to interrogate that +# type. +_JSONObject = Dict[str, object] # Names consist of letters, digits, -, and _, starting with a letter. @@ -315,9 +326,20 @@ def check_event(expr, info): def check_exprs(exprs): for expr_elem in exprs: - expr = expr_elem['expr'] - info = expr_elem['info'] - doc = expr_elem.get('doc') + # Expression + assert isinstance(expr_elem['expr'], dict) + for key in expr_elem['expr'].keys(): + assert isinstance(key, str) + expr: _JSONObject = expr_elem['expr'] + + # QAPISourceInfo + assert isinstance(expr_elem['info'], QAPISourceInfo) + info: QAPISourceInfo = expr_elem['info'] + + # Optional[QAPIDoc] + tmp = expr_elem.get('doc') + assert tmp is None or isinstance(tmp, QAPIDoc) + doc: Optional[QAPIDoc] = tmp if 'include' in expr: continue From b66c62a2d3318c5d968d5b95428efb74ca5d5702 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:20 -0400 Subject: [PATCH 05/25] qapi/expr.py: Add assertion for union type 'check_dict' mypy isn't fond of allowing you to check for bool membership in a collection of str elements. Guard this lookup for precisely when we were given a name. Signed-off-by: John Snow Reviewed-by: Eduardo Habkost Reviewed-by: Cleber Rosa Message-Id: <20210421182032.3521476-6-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index 06a0081001..3ab78a555d 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -194,7 +194,9 @@ def check_type(value, info, source, raise QAPISemError(info, "%s should be an object or type name" % source) - permissive = allow_dict in info.pragma.member_name_exceptions + permissive = False + if isinstance(allow_dict, str): + permissive = allow_dict in info.pragma.member_name_exceptions # value is a dictionary, check that each member is okay for (key, arg) in value.items(): From 926bb8add7c549496c612fcd4a32f3cf37883c2a Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:21 -0400 Subject: [PATCH 06/25] qapi/expr.py: move string check upwards in check_type For readability purposes only, shimmy the early return upwards to the top of the function, so cases proceed in order from least to most complex. Signed-off-by: John Snow Reviewed-by: Eduardo Habkost Reviewed-by: Cleber Rosa Message-Id: <20210421182032.3521476-7-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index 3ab78a555d..c0d18dcc01 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -171,6 +171,10 @@ def check_type(value, info, source, if value is None: return + # Type name + if isinstance(value, str): + return + # Array type if isinstance(value, list): if not allow_array: @@ -181,10 +185,6 @@ def check_type(value, info, source, source) return - # Type name - if isinstance(value, str): - return - # Anonymous type if not allow_dict: From 4918bb7defbdcb1e27cc2adf4e1604486d778ece Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:22 -0400 Subject: [PATCH 07/25] qapi/expr.py: Check type of union and alternate 'data' member Prior to this commit, specifying a non-object value here causes the QAPI parser to crash in expr.py with a stack trace with (likely) an AttributeError when we attempt to call that value's items() method. This member needs to be an object (Dict), and not anything else. Add a check for this with a nicer error message, and formalize that check with new test cases that exercise that error. Signed-off-by: John Snow Message-Id: <20210421182032.3521476-8-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 7 +++++++ tests/qapi-schema/alternate-data-invalid.err | 2 ++ tests/qapi-schema/alternate-data-invalid.json | 4 ++++ tests/qapi-schema/alternate-data-invalid.out | 0 tests/qapi-schema/meson.build | 2 ++ tests/qapi-schema/union-invalid-data.err | 2 ++ tests/qapi-schema/union-invalid-data.json | 6 ++++++ tests/qapi-schema/union-invalid-data.out | 0 8 files changed, 23 insertions(+) create mode 100644 tests/qapi-schema/alternate-data-invalid.err create mode 100644 tests/qapi-schema/alternate-data-invalid.json create mode 100644 tests/qapi-schema/alternate-data-invalid.out create mode 100644 tests/qapi-schema/union-invalid-data.err create mode 100644 tests/qapi-schema/union-invalid-data.json create mode 100644 tests/qapi-schema/union-invalid-data.out diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index c0d18dcc01..03624bdf3f 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -283,6 +283,9 @@ def check_union(expr, info): raise QAPISemError(info, "'discriminator' requires 'base'") check_name_is_str(discriminator, info, "'discriminator'") + if not isinstance(members, dict): + raise QAPISemError(info, "'data' must be an object") + for (key, value) in members.items(): source = "'data' member '%s'" % key if discriminator is None: @@ -298,6 +301,10 @@ def check_alternate(expr, info): if not members: raise QAPISemError(info, "'data' must not be empty") + + if not isinstance(members, dict): + raise QAPISemError(info, "'data' must be an object") + for (key, value) in members.items(): source = "'data' member '%s'" % key check_name_lower(key, info, source) diff --git a/tests/qapi-schema/alternate-data-invalid.err b/tests/qapi-schema/alternate-data-invalid.err new file mode 100644 index 0000000000..55f1033aef --- /dev/null +++ b/tests/qapi-schema/alternate-data-invalid.err @@ -0,0 +1,2 @@ +alternate-data-invalid.json: In alternate 'Alt': +alternate-data-invalid.json:2: 'data' must be an object diff --git a/tests/qapi-schema/alternate-data-invalid.json b/tests/qapi-schema/alternate-data-invalid.json new file mode 100644 index 0000000000..7d5d905581 --- /dev/null +++ b/tests/qapi-schema/alternate-data-invalid.json @@ -0,0 +1,4 @@ +# Alternate type requires an object for 'data' +{ 'alternate': 'Alt', + 'data': ['rubbish', 'nonsense'] +} diff --git a/tests/qapi-schema/alternate-data-invalid.out b/tests/qapi-schema/alternate-data-invalid.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/qapi-schema/meson.build b/tests/qapi-schema/meson.build index 8ba6917132..d7163e6601 100644 --- a/tests/qapi-schema/meson.build +++ b/tests/qapi-schema/meson.build @@ -14,6 +14,7 @@ schemas = [ 'alternate-conflict-string.json', 'alternate-conflict-bool-string.json', 'alternate-conflict-num-string.json', + 'alternate-data-invalid.json', 'alternate-empty.json', 'alternate-invalid-dict.json', 'alternate-nested.json', @@ -192,6 +193,7 @@ schemas = [ 'union-clash-branches.json', 'union-empty.json', 'union-invalid-base.json', + 'union-invalid-data.json', 'union-optional-branch.json', 'union-unknown.json', 'unknown-escape.json', diff --git a/tests/qapi-schema/union-invalid-data.err b/tests/qapi-schema/union-invalid-data.err new file mode 100644 index 0000000000..e26cf769a3 --- /dev/null +++ b/tests/qapi-schema/union-invalid-data.err @@ -0,0 +1,2 @@ +union-invalid-data.json: In union 'TestUnion': +union-invalid-data.json:2: 'data' must be an object diff --git a/tests/qapi-schema/union-invalid-data.json b/tests/qapi-schema/union-invalid-data.json new file mode 100644 index 0000000000..395ba24d39 --- /dev/null +++ b/tests/qapi-schema/union-invalid-data.json @@ -0,0 +1,6 @@ +# the union data type must be an object. +{ 'union': 'TestUnion', + 'base': 'int', + 'discriminator': 'int', + 'data': ['rubbish', 'nonsense'] +} diff --git a/tests/qapi-schema/union-invalid-data.out b/tests/qapi-schema/union-invalid-data.out new file mode 100644 index 0000000000..e69de29bb2 From 7a783ce5b5a3ac4762b866e22370dd4fb30b91bf Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:23 -0400 Subject: [PATCH 08/25] qapi/expr.py: Add casts in a few select cases Casts are instructions to the type checker only, they aren't "safe" and should probably be avoided in general. In this case, when we perform type checking on a nested structure, the type of each field does not "stick". (See PEP 647 for an example of "type narrowing" that does "stick". It is available in Python 3.10, so we can't use it yet.) We don't need to assert that something is a str if we've already checked or asserted that it is -- use a cast instead for these cases. Signed-off-by: John Snow Message-Id: <20210421182032.3521476-9-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index 03624bdf3f..f3a4a8536e 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -15,7 +15,7 @@ # See the COPYING file in the top-level directory. import re -from typing import Dict, Optional +from typing import Dict, Optional, cast from .common import c_name from .error import QAPISemError @@ -261,7 +261,7 @@ def check_enum(expr, info): def check_struct(expr, info): - name = expr['struct'] + name = cast(str, expr['struct']) # Checked in check_exprs members = expr['data'] check_type(members, info, "'data'", allow_dict=name) @@ -269,7 +269,7 @@ def check_struct(expr, info): def check_union(expr, info): - name = expr['union'] + name = cast(str, expr['union']) # Checked in check_exprs base = expr.get('base') discriminator = expr.get('discriminator') members = expr['data'] @@ -368,8 +368,8 @@ def check_exprs(exprs): else: raise QAPISemError(info, "expression is missing metatype") - name = expr[meta] - check_name_is_str(name, info, "'%s'" % meta) + check_name_is_str(expr[meta], info, "'%s'" % meta) + name = cast(str, expr[meta]) info.set_defn(meta, name) check_defn_name_str(name, info, meta) From 538cd41065ae5e506a1a07e866b1fd40b4b53d07 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:24 -0400 Subject: [PATCH 09/25] qapi/expr.py: Modify check_keys to accept any Collection This is a minor adjustment that lets parameters @required and @optional take tuple arguments, in particular (). Later patches will make use of that. (Iterable would also have worked, but Iterable also includes things like generator expressions which are consumed upon iteration, which would require a rewrite to make sure that each input was only traversed once. Collection implies the "can re-iterate" property.) Signed-off-by: John Snow Message-Id: <20210421182032.3521476-10-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index f3a4a8536e..396c8126d6 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -102,7 +102,7 @@ def check_keys(value, info, source, required, optional): "%s misses key%s %s" % (source, 's' if len(missing) > 1 else '', pprint(missing))) - allowed = set(required + optional) + allowed = set(required) | set(optional) unknown = set(value) - allowed if unknown: raise QAPISemError( From b9ad358aa057e83f8039a1e222d6941d2bf1f70a Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:25 -0400 Subject: [PATCH 10/25] qapi/expr.py: add type hint annotations Annotations do not change runtime behavior. This commit *only* adds annotations. Signed-off-by: John Snow Message-Id: <20210421182032.3521476-11-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 68 +++++++++++++++++++++++++++---------------- scripts/qapi/mypy.ini | 5 ---- 2 files changed, 43 insertions(+), 30 deletions(-) diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index 396c8126d6..4ebed4c488 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -15,7 +15,15 @@ # See the COPYING file in the top-level directory. import re -from typing import Dict, Optional, cast +from typing import ( + Collection, + Dict, + Iterable, + List, + Optional, + Union, + cast, +) from .common import c_name from .error import QAPISemError @@ -39,12 +47,14 @@ valid_name = re.compile(r'(__[a-z0-9.-]+_)?' r'([a-z][a-z0-9_-]*)$', re.IGNORECASE) -def check_name_is_str(name, info, source): +def check_name_is_str(name: object, + info: QAPISourceInfo, + source: str) -> None: if not isinstance(name, str): raise QAPISemError(info, "%s requires a string name" % source) -def check_name_str(name, info, source): +def check_name_str(name: str, info: QAPISourceInfo, source: str) -> str: # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty' # and 'q_obj_*' implicit type names. match = valid_name.match(name) @@ -53,16 +63,16 @@ def check_name_str(name, info, source): return match.group(3) -def check_name_upper(name, info, source): +def check_name_upper(name: str, info: QAPISourceInfo, source: str) -> None: stem = check_name_str(name, info, source) if re.search(r'[a-z-]', stem): raise QAPISemError( info, "name of %s must not use lowercase or '-'" % source) -def check_name_lower(name, info, source, - permit_upper=False, - permit_underscore=False): +def check_name_lower(name: str, info: QAPISourceInfo, source: str, + permit_upper: bool = False, + permit_underscore: bool = False) -> None: stem = check_name_str(name, info, source) if ((not permit_upper and re.search(r'[A-Z]', stem)) or (not permit_underscore and '_' in stem)): @@ -70,13 +80,13 @@ def check_name_lower(name, info, source, info, "name of %s must not use uppercase or '_'" % source) -def check_name_camel(name, info, source): +def check_name_camel(name: str, info: QAPISourceInfo, source: str) -> None: stem = check_name_str(name, info, source) if not re.match(r'[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$', stem): raise QAPISemError(info, "name of %s must use CamelCase" % source) -def check_defn_name_str(name, info, meta): +def check_defn_name_str(name: str, info: QAPISourceInfo, meta: str) -> None: if meta == 'event': check_name_upper(name, info, meta) elif meta == 'command': @@ -90,9 +100,13 @@ def check_defn_name_str(name, info, meta): info, "%s name should not end in '%s'" % (meta, name[-4:])) -def check_keys(value, info, source, required, optional): +def check_keys(value: _JSONObject, + info: QAPISourceInfo, + source: str, + required: Collection[str], + optional: Collection[str]) -> None: - def pprint(elems): + def pprint(elems: Iterable[str]) -> str: return ', '.join("'" + e + "'" for e in sorted(elems)) missing = set(required) - set(value) @@ -112,7 +126,7 @@ def check_keys(value, info, source, required, optional): pprint(unknown), pprint(allowed))) -def check_flags(expr, info): +def check_flags(expr: _JSONObject, info: QAPISourceInfo) -> None: for key in ['gen', 'success-response']: if key in expr and expr[key] is not False: raise QAPISemError( @@ -130,9 +144,9 @@ def check_flags(expr, info): "are incompatible") -def check_if(expr, info, source): +def check_if(expr: _JSONObject, info: QAPISourceInfo, source: str) -> None: - def check_if_str(ifcond): + def check_if_str(ifcond: object) -> None: if not isinstance(ifcond, str): raise QAPISemError( info, @@ -158,7 +172,7 @@ def check_if(expr, info, source): expr['if'] = [ifcond] -def normalize_members(members): +def normalize_members(members: object) -> None: if isinstance(members, dict): for key, arg in members.items(): if isinstance(arg, dict): @@ -166,8 +180,11 @@ def normalize_members(members): members[key] = {'type': arg} -def check_type(value, info, source, - allow_array=False, allow_dict=False): +def check_type(value: Optional[object], + info: QAPISourceInfo, + source: str, + allow_array: bool = False, + allow_dict: Union[bool, str] = False) -> None: if value is None: return @@ -214,7 +231,8 @@ def check_type(value, info, source, check_type(arg['type'], info, key_source, allow_array=True) -def check_features(features, info): +def check_features(features: Optional[object], + info: QAPISourceInfo) -> None: if features is None: return if not isinstance(features, list): @@ -231,7 +249,7 @@ def check_features(features, info): check_if(f, info, source) -def check_enum(expr, info): +def check_enum(expr: _JSONObject, info: QAPISourceInfo) -> None: name = expr['enum'] members = expr['data'] prefix = expr.get('prefix') @@ -260,7 +278,7 @@ def check_enum(expr, info): check_if(member, info, source) -def check_struct(expr, info): +def check_struct(expr: _JSONObject, info: QAPISourceInfo) -> None: name = cast(str, expr['struct']) # Checked in check_exprs members = expr['data'] @@ -268,7 +286,7 @@ def check_struct(expr, info): check_type(expr.get('base'), info, "'base'") -def check_union(expr, info): +def check_union(expr: _JSONObject, info: QAPISourceInfo) -> None: name = cast(str, expr['union']) # Checked in check_exprs base = expr.get('base') discriminator = expr.get('discriminator') @@ -296,7 +314,7 @@ def check_union(expr, info): check_type(value['type'], info, source, allow_array=not base) -def check_alternate(expr, info): +def check_alternate(expr: _JSONObject, info: QAPISourceInfo) -> None: members = expr['data'] if not members: @@ -313,7 +331,7 @@ def check_alternate(expr, info): check_type(value['type'], info, source) -def check_command(expr, info): +def check_command(expr: _JSONObject, info: QAPISourceInfo) -> None: args = expr.get('data') rets = expr.get('returns') boxed = expr.get('boxed', False) @@ -324,7 +342,7 @@ def check_command(expr, info): check_type(rets, info, "'returns'", allow_array=True) -def check_event(expr, info): +def check_event(expr: _JSONObject, info: QAPISourceInfo) -> None: args = expr.get('data') boxed = expr.get('boxed', False) @@ -333,7 +351,7 @@ def check_event(expr, info): check_type(args, info, "'data'", allow_dict=not boxed) -def check_exprs(exprs): +def check_exprs(exprs: List[_JSONObject]) -> List[_JSONObject]: for expr_elem in exprs: # Expression assert isinstance(expr_elem['expr'], dict) diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini index 0a000d58b3..7797c83432 100644 --- a/scripts/qapi/mypy.ini +++ b/scripts/qapi/mypy.ini @@ -8,11 +8,6 @@ disallow_untyped_defs = False disallow_incomplete_defs = False check_untyped_defs = False -[mypy-qapi.expr] -disallow_untyped_defs = False -disallow_incomplete_defs = False -check_untyped_defs = False - [mypy-qapi.parser] disallow_untyped_defs = False disallow_incomplete_defs = False From 210fd63104525b6e3154de529565258f19146f1a Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:26 -0400 Subject: [PATCH 11/25] qapi/expr.py: Consolidate check_if_str calls in check_if This is a small rewrite to address some minor style nits. Don't compare against the empty list to check for the empty condition, and move the normalization forward to unify the check on the now-normalized structure. With the check unified, the local nested function isn't needed anymore and can be brought down into the normal flow of the function. With the nesting level changed, shuffle the error strings around a bit to get them to fit in 79 columns. Note: although ifcond is typed as Sequence[str] elsewhere, we *know* that the parser will produce real, bona-fide lists. It's okay to check isinstance(ifcond, list) here. Signed-off-by: John Snow Message-Id: <20210421182032.3521476-12-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index 4ebed4c488..de7fc16fac 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -146,30 +146,29 @@ def check_flags(expr: _JSONObject, info: QAPISourceInfo) -> None: def check_if(expr: _JSONObject, info: QAPISourceInfo, source: str) -> None: - def check_if_str(ifcond: object) -> None: - if not isinstance(ifcond, str): + ifcond = expr.get('if') + if ifcond is None: + return + + if isinstance(ifcond, list): + if not ifcond: + raise QAPISemError( + info, "'if' condition [] of %s is useless" % source) + else: + # Normalize to a list + ifcond = expr['if'] = [ifcond] + + for elt in ifcond: + if not isinstance(elt, str): raise QAPISemError( info, "'if' condition of %s must be a string or a list of strings" % source) - if ifcond.strip() == '': + if not elt.strip(): raise QAPISemError( info, "'if' condition '%s' of %s makes no sense" - % (ifcond, source)) - - ifcond = expr.get('if') - if ifcond is None: - return - if isinstance(ifcond, list): - if ifcond == []: - raise QAPISemError( - info, "'if' condition [] of %s is useless" % source) - for elt in ifcond: - check_if_str(elt) - else: - check_if_str(ifcond) - expr['if'] = [ifcond] + % (elt, source)) def normalize_members(members: object) -> None: From e42648dccdd1defe8f35f247966cd7283f865cd6 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:27 -0400 Subject: [PATCH 12/25] qapi/expr.py: Remove single-letter variable Signed-off-by: John Snow Message-Id: <20210421182032.3521476-13-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index de7fc16fac..5e4d5f80aa 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -238,14 +238,14 @@ def check_features(features: Optional[object], raise QAPISemError(info, "'features' must be an array") features[:] = [f if isinstance(f, dict) else {'name': f} for f in features] - for f in features: + for feat in features: source = "'features' member" - assert isinstance(f, dict) - check_keys(f, info, source, ['name'], ['if']) - check_name_is_str(f['name'], info, source) - source = "%s '%s'" % (source, f['name']) - check_name_lower(f['name'], info, source) - check_if(f, info, source) + assert isinstance(feat, dict) + check_keys(feat, info, source, ['name'], ['if']) + check_name_is_str(feat['name'], info, source) + source = "%s '%s'" % (source, feat['name']) + check_name_str(feat['name'], info, source) + check_if(feat, info, source) def check_enum(expr: _JSONObject, info: QAPISourceInfo) -> None: From 328e8ca71abb9be7084ba16baad2a770c4e32d92 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:28 -0400 Subject: [PATCH 13/25] qapi/expr.py: enable pylint checks Signed-off-by: John Snow Tested-by: Eduardo Habkost Reviewed-by: Eduardo Habkost Reviewed-by: Cleber Rosa Tested-by: Cleber Rosa Message-Id: <20210421182032.3521476-14-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/pylintrc | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/qapi/pylintrc b/scripts/qapi/pylintrc index b9e077a164..fb0386d529 100644 --- a/scripts/qapi/pylintrc +++ b/scripts/qapi/pylintrc @@ -3,7 +3,6 @@ # Add files or directories matching the regex patterns to the ignore list. # The regex matches against base names, not paths. ignore-patterns=error.py, - expr.py, parser.py, schema.py, From 79e4fd14fb0f9e145413c6813dc541fdb50e3923 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:29 -0400 Subject: [PATCH 14/25] qapi/expr: Only explicitly prohibit 'Kind' nor 'List' for type names Per list review: qapi-code-gen.txt reserves suffixes Kind and List only for type names, but the code rejects them for events and commands, too. It turns out we reject them earlier anyway: In check_name_upper() for event names, and in check_name_lower() for command names. Still, adjust the code for clarity over what precisely we are guarding against. Signed-off-by: John Snow Message-Id: <20210421182032.3521476-15-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index 5e4d5f80aa..c33caf00d9 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -95,9 +95,9 @@ def check_defn_name_str(name: str, info: QAPISourceInfo, meta: str) -> None: permit_underscore=name in info.pragma.command_name_exceptions) else: check_name_camel(name, info, meta) - if name.endswith('Kind') or name.endswith('List'): - raise QAPISemError( - info, "%s name should not end in '%s'" % (meta, name[-4:])) + if name.endswith('Kind') or name.endswith('List'): + raise QAPISemError( + info, "%s name should not end in '%s'" % (meta, name[-4:])) def check_keys(value: _JSONObject, From a48653638fab9c9a9356b41d6e11544b2a7b330f Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:30 -0400 Subject: [PATCH 15/25] qapi/expr.py: Add docstrings Now with more :words:! Signed-off-by: John Snow Message-Id: <20210421182032.3521476-16-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 256 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 251 insertions(+), 5 deletions(-) diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index c33caf00d9..0b66d80842 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- # -# Check (context-free) QAPI schema expression structure -# # Copyright IBM, Corp. 2011 # Copyright (c) 2013-2019 Red Hat Inc. # @@ -14,6 +12,24 @@ # This work is licensed under the terms of the GNU GPL, version 2. # See the COPYING file in the top-level directory. +""" +Normalize and validate (context-free) QAPI schema expression structures. + +`QAPISchemaParser` parses a QAPI schema into abstract syntax trees +consisting of dict, list, str, bool, and int nodes. This module ensures +that these nested structures have the correct type(s) and key(s) where +appropriate for the QAPI context-free grammar. + +The QAPI schema expression language allows for certain syntactic sugar; +this module also handles the normalization process of these nested +structures. + +See `check_exprs` for the main entry point. + +See `schema.QAPISchema` for processing into native Python data +structures and contextual semantic validation. +""" + import re from typing import ( Collection, @@ -39,9 +55,7 @@ from .source import QAPISourceInfo _JSONObject = Dict[str, object] -# Names consist of letters, digits, -, and _, starting with a letter. -# An experimental name is prefixed with x-. A name of a downstream -# extension is prefixed with __RFQDN_. The latter prefix goes first. +# See check_name_str(), below. valid_name = re.compile(r'(__[a-z0-9.-]+_)?' r'(x-)?' r'([a-z][a-z0-9_-]*)$', re.IGNORECASE) @@ -50,11 +64,33 @@ valid_name = re.compile(r'(__[a-z0-9.-]+_)?' def check_name_is_str(name: object, info: QAPISourceInfo, source: str) -> None: + """ + Ensure that ``name`` is a ``str``. + + :raise QAPISemError: When ``name`` fails validation. + """ if not isinstance(name, str): raise QAPISemError(info, "%s requires a string name" % source) def check_name_str(name: str, info: QAPISourceInfo, source: str) -> str: + """ + Ensure that ``name`` is a valid QAPI name. + + A valid name consists of ASCII letters, digits, ``-``, and ``_``, + starting with a letter. It may be prefixed by a downstream prefix + of the form __RFQDN_, or the experimental prefix ``x-``. If both + prefixes are present, the __RFDQN_ prefix goes first. + + A valid name cannot start with ``q_``, which is reserved. + + :param name: Name to check. + :param info: QAPI schema source file information. + :param source: Error string describing what ``name`` belongs to. + + :raise QAPISemError: When ``name`` fails validation. + :return: The stem of the valid name, with no prefixes. + """ # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty' # and 'q_obj_*' implicit type names. match = valid_name.match(name) @@ -64,6 +100,19 @@ def check_name_str(name: str, info: QAPISourceInfo, source: str) -> str: def check_name_upper(name: str, info: QAPISourceInfo, source: str) -> None: + """ + Ensure that ``name`` is a valid event name. + + This means it must be a valid QAPI name as checked by + `check_name_str()`, but where the stem prohibits lowercase + characters and ``-``. + + :param name: Name to check. + :param info: QAPI schema source file information. + :param source: Error string describing what ``name`` belongs to. + + :raise QAPISemError: When ``name`` fails validation. + """ stem = check_name_str(name, info, source) if re.search(r'[a-z-]', stem): raise QAPISemError( @@ -73,6 +122,21 @@ def check_name_upper(name: str, info: QAPISourceInfo, source: str) -> None: def check_name_lower(name: str, info: QAPISourceInfo, source: str, permit_upper: bool = False, permit_underscore: bool = False) -> None: + """ + Ensure that ``name`` is a valid command or member name. + + This means it must be a valid QAPI name as checked by + `check_name_str()`, but where the stem prohibits uppercase + characters and ``_``. + + :param name: Name to check. + :param info: QAPI schema source file information. + :param source: Error string describing what ``name`` belongs to. + :param permit_upper: Additionally permit uppercase. + :param permit_underscore: Additionally permit ``_``. + + :raise QAPISemError: When ``name`` fails validation. + """ stem = check_name_str(name, info, source) if ((not permit_upper and re.search(r'[A-Z]', stem)) or (not permit_underscore and '_' in stem)): @@ -81,12 +145,39 @@ def check_name_lower(name: str, info: QAPISourceInfo, source: str, def check_name_camel(name: str, info: QAPISourceInfo, source: str) -> None: + """ + Ensure that ``name`` is a valid user-defined type name. + + This means it must be a valid QAPI name as checked by + `check_name_str()`, but where the stem must be in CamelCase. + + :param name: Name to check. + :param info: QAPI schema source file information. + :param source: Error string describing what ``name`` belongs to. + + :raise QAPISemError: When ``name`` fails validation. + """ stem = check_name_str(name, info, source) if not re.match(r'[A-Z][A-Za-z0-9]*[a-z][A-Za-z0-9]*$', stem): raise QAPISemError(info, "name of %s must use CamelCase" % source) def check_defn_name_str(name: str, info: QAPISourceInfo, meta: str) -> None: + """ + Ensure that ``name`` is a valid definition name. + + Based on the value of ``meta``, this means that: + - 'event' names adhere to `check_name_upper()`. + - 'command' names adhere to `check_name_lower()`. + - Else, meta is a type, and must pass `check_name_camel()`. + These names must not end with ``Kind`` nor ``List``. + + :param name: Name to check. + :param info: QAPI schema source file information. + :param meta: Meta-type name of the QAPI expression. + + :raise QAPISemError: When ``name`` fails validation. + """ if meta == 'event': check_name_upper(name, info, meta) elif meta == 'command': @@ -105,6 +196,17 @@ def check_keys(value: _JSONObject, source: str, required: Collection[str], optional: Collection[str]) -> None: + """ + Ensure that a dict has a specific set of keys. + + :param value: The dict to check. + :param info: QAPI schema source file information. + :param source: Error string describing this ``value``. + :param required: Keys that *must* be present. + :param optional: Keys that *may* be present. + + :raise QAPISemError: When unknown keys are present. + """ def pprint(elems: Iterable[str]) -> str: return ', '.join("'" + e + "'" for e in sorted(elems)) @@ -127,6 +229,16 @@ def check_keys(value: _JSONObject, def check_flags(expr: _JSONObject, info: QAPISourceInfo) -> None: + """ + Ensure flag members (if present) have valid values. + + :param expr: The expression to validate. + :param info: QAPI schema source file information. + + :raise QAPISemError: + When certain flags have an invalid value, or when + incompatible flags are present. + """ for key in ['gen', 'success-response']: if key in expr and expr[key] is not False: raise QAPISemError( @@ -145,7 +257,25 @@ def check_flags(expr: _JSONObject, info: QAPISourceInfo) -> None: def check_if(expr: _JSONObject, info: QAPISourceInfo, source: str) -> None: + """ + Normalize and validate the ``if`` member of an object. + The ``if`` member may be either a ``str`` or a ``List[str]``. + A ``str`` value will be normalized to ``List[str]``. + + :forms: + :sugared: ``Union[str, List[str]]`` + :canonical: ``List[str]`` + + :param expr: The expression containing the ``if`` member to validate. + :param info: QAPI schema source file information. + :param source: Error string describing ``expr``. + + :raise QAPISemError: + When the "if" member fails validation, or when there are no + non-empty conditions. + :return: None, ``expr`` is normalized in-place as needed. + """ ifcond = expr.get('if') if ifcond is None: return @@ -172,6 +302,21 @@ def check_if(expr: _JSONObject, info: QAPISourceInfo, source: str) -> None: def normalize_members(members: object) -> None: + """ + Normalize a "members" value. + + If ``members`` is a dict, for every value in that dict, if that + value is not itself already a dict, normalize it to + ``{'type': value}``. + + :forms: + :sugared: ``Dict[str, Union[str, TypeRef]]`` + :canonical: ``Dict[str, TypeRef]`` + + :param members: The members value to normalize. + + :return: None, ``members`` is normalized in-place as needed. + """ if isinstance(members, dict): for key, arg in members.items(): if isinstance(arg, dict): @@ -184,6 +329,26 @@ def check_type(value: Optional[object], source: str, allow_array: bool = False, allow_dict: Union[bool, str] = False) -> None: + """ + Normalize and validate the QAPI type of ``value``. + + Python types of ``str`` or ``None`` are always allowed. + + :param value: The value to check. + :param info: QAPI schema source file information. + :param source: Error string describing this ``value``. + :param allow_array: + Allow a ``List[str]`` of length 1, which indicates an array of + the type named by the list element. + :param allow_dict: + Allow a dict. Its members can be struct type members or union + branches. When the value of ``allow_dict`` is in pragma + ``member-name-exceptions``, the dict's keys may violate the + member naming rules. The dict members are normalized in place. + + :raise QAPISemError: When ``value`` fails validation. + :return: None, ``value`` is normalized in-place as needed. + """ if value is None: return @@ -232,6 +397,22 @@ def check_type(value: Optional[object], def check_features(features: Optional[object], info: QAPISourceInfo) -> None: + """ + Normalize and validate the ``features`` member. + + ``features`` may be a ``list`` of either ``str`` or ``dict``. + Any ``str`` element will be normalized to ``{'name': element}``. + + :forms: + :sugared: ``List[Union[str, Feature]]`` + :canonical: ``List[Feature]`` + + :param features: The features member value to validate. + :param info: QAPI schema source file information. + + :raise QAPISemError: When ``features`` fails validation. + :return: None, ``features`` is normalized in-place as needed. + """ if features is None: return if not isinstance(features, list): @@ -249,6 +430,15 @@ def check_features(features: Optional[object], def check_enum(expr: _JSONObject, info: QAPISourceInfo) -> None: + """ + Normalize and validate this expression as an ``enum`` definition. + + :param expr: The expression to validate. + :param info: QAPI schema source file information. + + :raise QAPISemError: When ``expr`` is not a valid ``enum``. + :return: None, ``expr`` is normalized in-place as needed. + """ name = expr['enum'] members = expr['data'] prefix = expr.get('prefix') @@ -278,6 +468,15 @@ def check_enum(expr: _JSONObject, info: QAPISourceInfo) -> None: def check_struct(expr: _JSONObject, info: QAPISourceInfo) -> None: + """ + Normalize and validate this expression as a ``struct`` definition. + + :param expr: The expression to validate. + :param info: QAPI schema source file information. + + :raise QAPISemError: When ``expr`` is not a valid ``struct``. + :return: None, ``expr`` is normalized in-place as needed. + """ name = cast(str, expr['struct']) # Checked in check_exprs members = expr['data'] @@ -286,6 +485,15 @@ def check_struct(expr: _JSONObject, info: QAPISourceInfo) -> None: def check_union(expr: _JSONObject, info: QAPISourceInfo) -> None: + """ + Normalize and validate this expression as a ``union`` definition. + + :param expr: The expression to validate. + :param info: QAPI schema source file information. + + :raise QAPISemError: when ``expr`` is not a valid ``union``. + :return: None, ``expr`` is normalized in-place as needed. + """ name = cast(str, expr['union']) # Checked in check_exprs base = expr.get('base') discriminator = expr.get('discriminator') @@ -314,6 +522,15 @@ def check_union(expr: _JSONObject, info: QAPISourceInfo) -> None: def check_alternate(expr: _JSONObject, info: QAPISourceInfo) -> None: + """ + Normalize and validate this expression as an ``alternate`` definition. + + :param expr: The expression to validate. + :param info: QAPI schema source file information. + + :raise QAPISemError: When ``expr`` is not a valid ``alternate``. + :return: None, ``expr`` is normalized in-place as needed. + """ members = expr['data'] if not members: @@ -331,6 +548,15 @@ def check_alternate(expr: _JSONObject, info: QAPISourceInfo) -> None: def check_command(expr: _JSONObject, info: QAPISourceInfo) -> None: + """ + Normalize and validate this expression as a ``command`` definition. + + :param expr: The expression to validate. + :param info: QAPI schema source file information. + + :raise QAPISemError: When ``expr`` is not a valid ``command``. + :return: None, ``expr`` is normalized in-place as needed. + """ args = expr.get('data') rets = expr.get('returns') boxed = expr.get('boxed', False) @@ -342,6 +568,15 @@ def check_command(expr: _JSONObject, info: QAPISourceInfo) -> None: def check_event(expr: _JSONObject, info: QAPISourceInfo) -> None: + """ + Normalize and validate this expression as an ``event`` definition. + + :param expr: The expression to validate. + :param info: QAPI schema source file information. + + :raise QAPISemError: When ``expr`` is not a valid ``event``. + :return: None, ``expr`` is normalized in-place as needed. + """ args = expr.get('data') boxed = expr.get('boxed', False) @@ -351,6 +586,17 @@ def check_event(expr: _JSONObject, info: QAPISourceInfo) -> None: def check_exprs(exprs: List[_JSONObject]) -> List[_JSONObject]: + """ + Validate and normalize a list of parsed QAPI schema expressions. + + This function accepts a list of expressions and metadata as returned + by the parser. It destructively normalizes the expressions in-place. + + :param exprs: The list of expressions to normalize and validate. + + :raise QAPISemError: When any expression fails validation. + :return: The same list of expressions (now modified). + """ for expr_elem in exprs: # Expression assert isinstance(expr_elem['expr'], dict) From eab99939a7cd289a8b50c2e7ef6a38ca2706a46c Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:31 -0400 Subject: [PATCH 16/25] qapi/expr.py: Use tuples instead of lists for static data It is -- maybe -- possibly -- three nanoseconds faster. Signed-off-by: John Snow Reviewed-by: Eduardo Habkost Reviewed-by: Cleber Rosa Message-Id: <20210421182032.3521476-17-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index 0b66d80842..225a82e20d 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -239,11 +239,11 @@ def check_flags(expr: _JSONObject, info: QAPISourceInfo) -> None: When certain flags have an invalid value, or when incompatible flags are present. """ - for key in ['gen', 'success-response']: + for key in ('gen', 'success-response'): if key in expr and expr[key] is not False: raise QAPISemError( info, "flag '%s' may only use false value" % key) - for key in ['boxed', 'allow-oob', 'allow-preconfig', 'coroutine']: + for key in ('boxed', 'allow-oob', 'allow-preconfig', 'coroutine'): if key in expr and expr[key] is not True: raise QAPISemError( info, "flag '%s' may only use true value" % key) From e81718c698a9f1a1d98edd605f508dadbffe0d4d Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 14:20:32 -0400 Subject: [PATCH 17/25] qapi/expr: Update authorship and copyright information Signed-off-by: John Snow Message-Id: <20210421182032.3521476-18-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/expr.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/qapi/expr.py b/scripts/qapi/expr.py index 225a82e20d..496f7e0333 100644 --- a/scripts/qapi/expr.py +++ b/scripts/qapi/expr.py @@ -1,13 +1,14 @@ # -*- coding: utf-8 -*- # # Copyright IBM, Corp. 2011 -# Copyright (c) 2013-2019 Red Hat Inc. +# Copyright (c) 2013-2021 Red Hat Inc. # # Authors: # Anthony Liguori # Markus Armbruster # Eric Blake # Marc-André Lureau +# John Snow # # This work is licensed under the terms of the GNU GPL, version 2. # See the COPYING file in the top-level directory. From 46f49468c690ff015a5b5346a279845f5e55369e Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 15:22:26 -0400 Subject: [PATCH 18/25] qapi/error: Repurpose QAPIError as an abstract base exception class Rename QAPIError to QAPISourceError, and then create a new QAPIError class that serves as the basis for all of our other custom exceptions, without specifying any class properties. This leaves QAPIError as a package-wide error class that's suitable for any current or future errors. (Right now, we don't have any errors that DON'T also want to specify a Source location, but this MAY change. In these cases, a common abstract ancestor would be desired.) Add docstrings to explain the intended function of each error class. Signed-off-by: John Snow Message-Id: <20210421192233.3542904-2-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- docs/sphinx/qapidoc.py | 3 ++- scripts/qapi/error.py | 11 +++++++++-- scripts/qapi/schema.py | 5 +++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/docs/sphinx/qapidoc.py b/docs/sphinx/qapidoc.py index b7a2d39c10..87c67ab23f 100644 --- a/docs/sphinx/qapidoc.py +++ b/docs/sphinx/qapidoc.py @@ -34,7 +34,8 @@ from sphinx.errors import ExtensionError from sphinx.util.nodes import nested_parse_with_titles import sphinx from qapi.gen import QAPISchemaVisitor -from qapi.schema import QAPIError, QAPISemError, QAPISchema +from qapi.error import QAPIError, QAPISemError +from qapi.schema import QAPISchema # Sphinx up to 1.6 uses AutodocReporter; 1.7 and later diff --git a/scripts/qapi/error.py b/scripts/qapi/error.py index ae60d9e2fe..126dda7c9b 100644 --- a/scripts/qapi/error.py +++ b/scripts/qapi/error.py @@ -13,6 +13,11 @@ class QAPIError(Exception): + """Base class for all exceptions from the QAPI package.""" + + +class QAPISourceError(QAPIError): + """Error class for all exceptions identifying a source location.""" def __init__(self, info, col, msg): Exception.__init__(self) self.info = info @@ -27,7 +32,8 @@ class QAPIError(Exception): return loc + ': ' + self.msg -class QAPIParseError(QAPIError): +class QAPIParseError(QAPISourceError): + """Error class for all QAPI schema parsing errors.""" def __init__(self, parser, msg): col = 1 for ch in parser.src[parser.line_pos:parser.pos]: @@ -38,6 +44,7 @@ class QAPIParseError(QAPIError): super().__init__(parser.info, col, msg) -class QAPISemError(QAPIError): +class QAPISemError(QAPISourceError): + """Error class for semantic QAPI errors.""" def __init__(self, info, msg): super().__init__(info, None, msg) diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py index 703b446fd2..c277fcacc5 100644 --- a/scripts/qapi/schema.py +++ b/scripts/qapi/schema.py @@ -20,7 +20,7 @@ import re from typing import Optional from .common import POINTER_SUFFIX, c_name -from .error import QAPIError, QAPISemError +from .error import QAPISemError, QAPISourceError from .expr import check_exprs from .parser import QAPISchemaParser @@ -875,7 +875,8 @@ class QAPISchema: other_ent = self._entity_dict.get(ent.name) if other_ent: if other_ent.info: - where = QAPIError(other_ent.info, None, "previous definition") + where = QAPISourceError(other_ent.info, None, + "previous definition") raise QAPISemError( ent.info, "'%s' is already defined\n%s" % (ent.name, where)) From b54e07cc46064e79de275c7ea26d90a51913a9ea Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 15:22:27 -0400 Subject: [PATCH 19/25] qapi/error: Use Python3-style super() Missed in commit 2cae67bcb5 "qapi: Use super() now we have Python 3". Signed-off-by: John Snow Reviewed-by: Markus Armbruster Message-Id: <20210421192233.3542904-3-jsnow@redhat.com> Signed-off-by: Markus Armbruster --- scripts/qapi/error.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/qapi/error.py b/scripts/qapi/error.py index 126dda7c9b..38bd7c4dd6 100644 --- a/scripts/qapi/error.py +++ b/scripts/qapi/error.py @@ -19,7 +19,7 @@ class QAPIError(Exception): class QAPISourceError(QAPIError): """Error class for all exceptions identifying a source location.""" def __init__(self, info, col, msg): - Exception.__init__(self) + super().__init__() self.info = info self.col = col self.msg = msg From 86cc2ff65a4764ade26c7741c7c05f23e7efa95c Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 15:22:28 -0400 Subject: [PATCH 20/25] qapi/error: Make QAPISourceError 'col' parameter optional It's already treated as optional, with one direct caller and some subclass callers passing 'None'. Make it officially optional, which requires moving the position of the argument to come after all required parameters. QAPISemError becomes functionally identical to QAPISourceError. Keep the name to preserve its semantic meaning and avoid code churn, but remove the now-useless __init__ wrapper. Signed-off-by: John Snow Message-Id: <20210421192233.3542904-4-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/error.py | 8 +++----- scripts/qapi/schema.py | 3 +-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/scripts/qapi/error.py b/scripts/qapi/error.py index 38bd7c4dd6..d179a3bd0c 100644 --- a/scripts/qapi/error.py +++ b/scripts/qapi/error.py @@ -18,11 +18,11 @@ class QAPIError(Exception): class QAPISourceError(QAPIError): """Error class for all exceptions identifying a source location.""" - def __init__(self, info, col, msg): + def __init__(self, info, msg, col=None): super().__init__() self.info = info - self.col = col self.msg = msg + self.col = col def __str__(self): loc = str(self.info) @@ -41,10 +41,8 @@ class QAPIParseError(QAPISourceError): col = (col + 7) % 8 + 1 else: col += 1 - super().__init__(parser.info, col, msg) + super().__init__(parser.info, msg, col) class QAPISemError(QAPISourceError): """Error class for semantic QAPI errors.""" - def __init__(self, info, msg): - super().__init__(info, None, msg) diff --git a/scripts/qapi/schema.py b/scripts/qapi/schema.py index c277fcacc5..3a4172fb74 100644 --- a/scripts/qapi/schema.py +++ b/scripts/qapi/schema.py @@ -875,8 +875,7 @@ class QAPISchema: other_ent = self._entity_dict.get(ent.name) if other_ent: if other_ent.info: - where = QAPISourceError(other_ent.info, None, - "previous definition") + where = QAPISourceError(other_ent.info, "previous definition") raise QAPISemError( ent.info, "'%s' is already defined\n%s" % (ent.name, where)) From ac89761179ed6e3165a63ad68759f77f33bace30 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 15:22:29 -0400 Subject: [PATCH 21/25] qapi/error: assert QAPISourceInfo is not None Built-in stuff is not parsed from a source file, and therefore have no QAPISourceInfo. If such None info was used for reporting an error, built-in stuff would be broken. Programming error. Instead of reporting a confusing error with bogus source location then, we better crash. We currently crash only if self.col was set. Assert that self.info is not None in order to crash reliably. We can not yet change the type of the initializer to prove this cannot happen at static analysis time before the remainder of the code is fully typed. Signed-off-by: John Snow Message-Id: <20210421192233.3542904-5-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/error.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/qapi/error.py b/scripts/qapi/error.py index d179a3bd0c..d0bc7af6e7 100644 --- a/scripts/qapi/error.py +++ b/scripts/qapi/error.py @@ -25,6 +25,7 @@ class QAPISourceError(QAPIError): self.col = col def __str__(self): + assert self.info is not None loc = str(self.info) if self.col is not None: assert self.info.line is not None From ac6a7d8884762d27cc2dde5a5c6e793cc18fc4d9 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 15:22:30 -0400 Subject: [PATCH 22/25] qapi/error.py: move QAPIParseError to parser.py Keeping it in error.py will create some cyclic import problems when we add types to the QAPISchemaParser. Callers don't need to know the details of QAPIParseError unless they are parsing or dealing directly with the parser, so this won't create any harsh new requirements for callers in the general case. Update error.py with a little docstring that gives a nod to where the error may now be found. Signed-off-by: John Snow Message-Id: <20210421192233.3542904-6-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/error.py | 22 ++++++++-------------- scripts/qapi/parser.py | 14 +++++++++++++- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/scripts/qapi/error.py b/scripts/qapi/error.py index d0bc7af6e7..6723c5a9d9 100644 --- a/scripts/qapi/error.py +++ b/scripts/qapi/error.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- # -# QAPI error classes -# # Copyright (c) 2017-2019 Red Hat Inc. # # Authors: @@ -11,6 +9,14 @@ # This work is licensed under the terms of the GNU GPL, version 2. # See the COPYING file in the top-level directory. +""" +QAPI error classes + +Common error classes used throughout the package. Additional errors may +be defined in other modules. At present, `QAPIParseError` is defined in +parser.py. +""" + class QAPIError(Exception): """Base class for all exceptions from the QAPI package.""" @@ -33,17 +39,5 @@ class QAPISourceError(QAPIError): return loc + ': ' + self.msg -class QAPIParseError(QAPISourceError): - """Error class for all QAPI schema parsing errors.""" - def __init__(self, parser, msg): - col = 1 - for ch in parser.src[parser.line_pos:parser.pos]: - if ch == '\t': - col = (col + 7) % 8 + 1 - else: - col += 1 - super().__init__(parser.info, msg, col) - - class QAPISemError(QAPISourceError): """Error class for semantic QAPI errors.""" diff --git a/scripts/qapi/parser.py b/scripts/qapi/parser.py index 58267c3db9..ca5e8e18e0 100644 --- a/scripts/qapi/parser.py +++ b/scripts/qapi/parser.py @@ -18,10 +18,22 @@ from collections import OrderedDict import os import re -from .error import QAPIParseError, QAPISemError +from .error import QAPISemError, QAPISourceError from .source import QAPISourceInfo +class QAPIParseError(QAPISourceError): + """Error class for all QAPI schema parsing errors.""" + def __init__(self, parser, msg): + col = 1 + for ch in parser.src[parser.line_pos:parser.pos]: + if ch == '\t': + col = (col + 7) % 8 + 1 + else: + col += 1 + super().__init__(parser.info, msg, col) + + class QAPISchemaParser: def __init__(self, fname, previously_included=None, incl_info=None): From 92870cf3afe42c0f2103ff3f5e4e7edd99549040 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 15:22:31 -0400 Subject: [PATCH 23/25] qapi/error.py: enable pylint checks Signed-off-by: John Snow Message-Id: <20210421192233.3542904-7-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/pylintrc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/qapi/pylintrc b/scripts/qapi/pylintrc index fb0386d529..88efbf71cb 100644 --- a/scripts/qapi/pylintrc +++ b/scripts/qapi/pylintrc @@ -2,8 +2,7 @@ # Add files or directories matching the regex patterns to the ignore list. # The regex matches against base names, not paths. -ignore-patterns=error.py, - parser.py, +ignore-patterns=parser.py, schema.py, From 30d0a016e965796b41ac545b3d527f8080292869 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 15:22:32 -0400 Subject: [PATCH 24/25] qapi/error: Add type hints No functional change. Note: QAPISourceError's info parameter is Optional[] because schema.py treats the info property of its various classes as Optional to accommodate built-in types, which have no source. See prior commit 'qapi/error: assert QAPISourceInfo is not None'. Signed-off-by: John Snow Message-Id: <20210421192233.3542904-8-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/error.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/qapi/error.py b/scripts/qapi/error.py index 6723c5a9d9..e35e4ddb26 100644 --- a/scripts/qapi/error.py +++ b/scripts/qapi/error.py @@ -17,6 +17,10 @@ be defined in other modules. At present, `QAPIParseError` is defined in parser.py. """ +from typing import Optional + +from .source import QAPISourceInfo + class QAPIError(Exception): """Base class for all exceptions from the QAPI package.""" @@ -24,13 +28,16 @@ class QAPIError(Exception): class QAPISourceError(QAPIError): """Error class for all exceptions identifying a source location.""" - def __init__(self, info, msg, col=None): + def __init__(self, + info: Optional[QAPISourceInfo], + msg: str, + col: Optional[int] = None): super().__init__() self.info = info self.msg = msg self.col = col - def __str__(self): + def __str__(self) -> str: assert self.info is not None loc = str(self.info) if self.col is not None: From b54626e0b8f423e91b2e31fa7741e4954cebd2d6 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 21 Apr 2021 15:22:33 -0400 Subject: [PATCH 25/25] qapi/error.py: enable mypy checks Signed-off-by: John Snow Message-Id: <20210421192233.3542904-9-jsnow@redhat.com> Reviewed-by: Markus Armbruster Signed-off-by: Markus Armbruster --- scripts/qapi/mypy.ini | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scripts/qapi/mypy.ini b/scripts/qapi/mypy.ini index 7797c83432..54ca4483d6 100644 --- a/scripts/qapi/mypy.ini +++ b/scripts/qapi/mypy.ini @@ -3,11 +3,6 @@ strict = True disallow_untyped_calls = False python_version = 3.6 -[mypy-qapi.error] -disallow_untyped_defs = False -disallow_incomplete_defs = False -check_untyped_defs = False - [mypy-qapi.parser] disallow_untyped_defs = False disallow_incomplete_defs = False