diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0784242d3..6cf00cf28 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -27,10 +27,11 @@ Refactoring: Features: -* Add a new ``unknown`` parameter to ``Parser.parse``, ``Parser.use_args``, and - ``Parser.use_kwargs``. When set, it will be passed to the ``Schema.load`` - call. If set to ``None`` (the default), no value is passed, so the schema's - ``unknown`` behavior is used. +* Add ``unknown`` as a parameter to ``Parser.parse``, ``Parser.use_args``, + ``Parser.use_kwargs``, and parser instantiation. When set, it will be passed + to ``Schema.load``. When not set, the value passed will depend on the parser's + settings. If set to ``None``, the schema's default behavior will be used (i.e. + no value is passed to ``Schema.load``) and parser settings will be ignored. This allows usages like @@ -45,10 +46,9 @@ This allows usages like def foo(q1, q2): ... -* Add the ability to set defaults for ``unknown`` on either a Parser instance - or Parser class. Set ``Parser.DEFAULT_UNKNOWN`` on a parser class to apply a value - to any new parser instances created from that class, or set ``unknown`` during - ``Parser`` initialization. +* Defaults for ``unknown`` may be customized on parser classes via + ``Parser.DEFAULT_UNKNOWN_BY_LOCATION``, which maps location names to values + to use. Usages are varied, but include @@ -57,15 +57,27 @@ Usages are varied, but include import marshmallow as ma from webargs.flaskparser import FlaskParser - parser = FlaskParser(unknown=ma.INCLUDE) - # as well as... class MyParser(FlaskParser): - DEFAULT_UNKNOWN = ma.INCLUDE + DEFAULT_UNKNOWN_BY_LOCATION = {"query": ma.INCLUDE} parser = MyParser() +Setting the ``unknown`` value for a Parser instance has higher precedence. So + +.. code-block:: python + + parser = MyParser(unknown=ma.RAISE) + +will always pass ``RAISE``, even when the location is ``query``. + +* By default, webargs will pass ``unknown=EXCLUDE`` for all locations except + for request bodies (``json``, ``form``, and ``json_or_form``) and path + parameters. Request bodies and path parameters will pass ``unknown=RAISE``. + This behavior is defined by the default value for + ``DEFAULT_UNKNOWN_BY_LOCATION``. + Changes: * Registered `error_handler` callbacks are required to raise an exception. diff --git a/docs/advanced.rst b/docs/advanced.rst index 108eda937..45ddd5e56 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -128,6 +128,122 @@ When you need more flexibility in defining input schemas, you can pass a marshma # ... +Setting `unknown` +----------------- + +webargs supports several ways of setting and passing the `unknown` parameter +for `handling unknown fields `_. + +You can pass `unknown=...` as a parameter to any of +`Parser.parse `, +`Parser.use_args `, and +`Parser.use_kwargs `. + + +.. note:: + + The `unknown` value is passed to the schema's `load()` call. It therefore + only applies to the top layer when nesting is used. To control `unknown` at + multiple layers of a nested schema, you must use other mechanisms, like + the `unknown` argument to `fields.Nested`. + +Default `unknown` ++++++++++++++++++ + +By default, webargs will pass `unknown=marshmallow.EXCLUDE` except when the +location is `json`, `form`, `json_or_form`, `path`, or `path`. In those cases, +it uses `unknown=marshmallow.RAISE` instead. + +You can change these defaults by overriding `DEFAULT_UNKNOWN_BY_LOCATION`. +This is a mapping of locations to values to pass. + +For example, + +.. code-block:: python + + from flask import Flask + from marshmallow import EXCLUDE, fields + from webargs.flaskparser import FlaskParser + + app = Flask(__name__) + + + class Parser(FlaskParser): + DEFAULT_UNKNOWN_BY_LOCATION = {"query": EXCLUDE} + + + parser = Parser() + + + # location is "query", which is listed in DEFAULT_UNKNOWN_BY_LOCATION, + # so EXCLUDE will be used + @app.route("/", methods=["GET"]) + @parser.use_args({"foo": fields.Int()}, location="query") + def get(self, args): + return f"foo x 2 = {args['foo'] * 2}" + + + # location is "json", which is not in DEFAULT_UNKNOWN_BY_LOCATION, + # so no value will be passed for `unknown` + @app.route("/", methods=["POST"]) + @parser.use_args({"foo": fields.Int(), "bar": fields.Int()}, location="json") + def post(self, args): + return f"foo x bar = {args['foo'] * args['bar']}" + + +You can also define a default at parser instantiation, which will take +precedence over these defaults, as in + +.. code-block:: python + + from marshmallow import INCLUDE + + parser = Parser(unknown=INCLUDE) + + # because `unknown` is set on the parser, `DEFAULT_UNKNOWN_BY_LOCATION` has + # effect and `INCLUDE` will always be used + @app.route("/", methods=["POST"]) + @parser.use_args({"foo": fields.Int(), "bar": fields.Int()}, location="json") + def post(self, args): + unexpected_args = [k for k in args.keys() if k not in ("foo", "bar")] + return f"foo x bar = {args['foo'] * args['bar']}; unexpected args={unexpected_args}" + +Using Schema-Specfied `unknown` ++++++++++++++++++++++++++++++++ + +If you wish to use the value of `unknown` specified by a schema, simply pass +``unknown=None``. This will disable webargs' automatic passing of values for +``unknown``. For example, + +.. code-block:: python + + from flask import Flask + from marshmallow import Schema, fields, EXCLUDE, missing + from webargs.flaskparser import use_args + + + class RectangleSchema(Schema): + length = fields.Float() + width = fields.Float() + + class Meta: + unknown = EXCLUDE + + + app = Flask(__name__) + + # because unknown=None was passed, no value is passed during schema loading + # as a result, the schema's behavior (EXCLUDE) is used + @app.route("/", methods=["POST"]) + @use_args(RectangleSchema(), location="json", unknown=None) + def get(self, args): + return f"area = {args['length'] * args['width']}" + + +You can also set ``unknown=None`` when instantiating a parser to make this +behavior the default for a parser. + + When to avoid `use_kwargs` -------------------------- diff --git a/src/webargs/aiohttpparser.py b/src/webargs/aiohttpparser.py index 2242c026d..ad871d0d8 100644 --- a/src/webargs/aiohttpparser.py +++ b/src/webargs/aiohttpparser.py @@ -27,7 +27,7 @@ def index(request, args): from aiohttp import web from aiohttp.web import Request from aiohttp import web_exceptions -from marshmallow import Schema, ValidationError +from marshmallow import Schema, ValidationError, RAISE from webargs import core from webargs.core import json @@ -72,6 +72,11 @@ def _find_exceptions() -> None: class AIOHTTPParser(AsyncParser): """aiohttp request argument parser.""" + DEFAULT_UNKNOWN_BY_LOCATION = { + "match_info": RAISE, + "path": RAISE, + **core.Parser.DEFAULT_UNKNOWN_BY_LOCATION, + } __location_map__ = dict( match_info="load_match_info", path="load_match_info", diff --git a/src/webargs/asyncparser.py b/src/webargs/asyncparser.py index b066932eb..cb789e7ef 100644 --- a/src/webargs/asyncparser.py +++ b/src/webargs/asyncparser.py @@ -28,7 +28,7 @@ async def parse( req: Request = None, *, location: str = None, - unknown: str = None, + unknown: str = core._UNKNOWN_DEFAULT_PARAM, validate: Validate = None, error_status_code: typing.Union[int, None] = None, error_headers: typing.Union[typing.Mapping[str, str], None] = None @@ -39,7 +39,15 @@ async def parse( """ req = req if req is not None else self.get_default_request() location = location or self.location - unknown = unknown or self.unknown + unknown = ( + unknown + if unknown != core._UNKNOWN_DEFAULT_PARAM + else ( + self.unknown + if self.unknown != core._UNKNOWN_DEFAULT_PARAM + else self.DEFAULT_UNKNOWN_BY_LOCATION.get(location) + ) + ) load_kwargs = {"unknown": unknown} if req is None: raise ValueError("Must pass req object") @@ -113,7 +121,7 @@ def use_args( req: typing.Optional[Request] = None, *, location: str = None, - unknown=None, + unknown=core._UNKNOWN_DEFAULT_PARAM, as_kwargs: bool = False, validate: Validate = None, error_status_code: typing.Optional[int] = None, diff --git a/src/webargs/core.py b/src/webargs/core.py index 9e468672d..a8be97ade 100644 --- a/src/webargs/core.py +++ b/src/webargs/core.py @@ -23,6 +23,10 @@ ] +# a value used as the default for arguments, so that when `None` is passed, it +# can be distinguished from the default value +_UNKNOWN_DEFAULT_PARAM = "_default" + DEFAULT_VALIDATION_STATUS = 422 # type: int @@ -97,15 +101,27 @@ class Parser: etc. :param str location: Default location to use for data - :param str unknown: Default value for ``unknown`` in ``parse``, - ``use_args``, and ``use_kwargs`` + :param str unknown: A default value to pass for ``unknown`` when calling the + schema's ``load`` method. Defaults to EXCLUDE for non-body + locations and RAISE for request bodies. Pass ``None`` to use the + schema's setting instead. :param callable error_handler: Custom error handler function. """ #: Default location to check for data DEFAULT_LOCATION = "json" #: Default value to use for 'unknown' on schema load - DEFAULT_UNKNOWN = None + # on a per-location basis + DEFAULT_UNKNOWN_BY_LOCATION = { + "json": ma.RAISE, + "form": ma.RAISE, + "json_or_form": ma.RAISE, + "querystring": ma.EXCLUDE, + "query": ma.EXCLUDE, + "headers": ma.EXCLUDE, + "cookies": ma.EXCLUDE, + "files": ma.EXCLUDE, + } #: The marshmallow Schema class to use when creating new schemas DEFAULT_SCHEMA_CLASS = ma.Schema #: Default status code to return for validation errors @@ -126,12 +142,17 @@ class Parser: } def __init__( - self, location=None, *, unknown=None, error_handler=None, schema_class=None + self, + location=None, + *, + unknown=_UNKNOWN_DEFAULT_PARAM, + error_handler=None, + schema_class=None ): self.location = location or self.DEFAULT_LOCATION self.error_callback = _callable_or_raise(error_handler) self.schema_class = schema_class or self.DEFAULT_SCHEMA_CLASS - self.unknown = unknown or self.DEFAULT_UNKNOWN + self.unknown = unknown def _get_loader(self, location): """Get the loader function for the given location. @@ -219,7 +240,7 @@ def parse( req=None, *, location=None, - unknown=None, + unknown=_UNKNOWN_DEFAULT_PARAM, validate=None, error_status_code=None, error_headers=None @@ -235,7 +256,9 @@ def parse( default, that means one of ``('json', 'query', 'querystring', 'form', 'headers', 'cookies', 'files', 'json_or_form')``. :param str unknown: A value to pass for ``unknown`` when calling the - schema's ``load`` method. + schema's ``load`` method. Defaults to EXCLUDE for non-body + locations and RAISE for request bodies. Pass ``None`` to use the + schema's setting instead. :param callable validate: Validation function or list of validation functions that receives the dictionary of parsed arguments. Validator either returns a boolean or raises a :exc:`ValidationError`. @@ -248,8 +271,17 @@ def parse( """ req = req if req is not None else self.get_default_request() location = location or self.location - unknown = unknown or self.unknown - load_kwargs = {"unknown": unknown} + # precedence order: explicit, instance setting, default per location + unknown = ( + unknown + if unknown != _UNKNOWN_DEFAULT_PARAM + else ( + self.unknown + if self.unknown != _UNKNOWN_DEFAULT_PARAM + else self.DEFAULT_UNKNOWN_BY_LOCATION.get(location) + ) + ) + load_kwargs = {"unknown": unknown} if unknown else {} if req is None: raise ValueError("Must pass req object") data = None @@ -311,7 +343,7 @@ def use_args( req=None, *, location=None, - unknown=None, + unknown=_UNKNOWN_DEFAULT_PARAM, as_kwargs=False, validate=None, error_status_code=None, diff --git a/src/webargs/flaskparser.py b/src/webargs/flaskparser.py index a74e0342f..4fbe15c8a 100644 --- a/src/webargs/flaskparser.py +++ b/src/webargs/flaskparser.py @@ -23,6 +23,8 @@ def user_detail(args, uid): import flask from werkzeug.exceptions import HTTPException +import marshmallow as ma + from webargs import core from webargs.multidictproxy import MultiDictProxy @@ -48,6 +50,11 @@ def is_json_request(req): class FlaskParser(core.Parser): """Flask request argument parser.""" + DEFAULT_UNKNOWN_BY_LOCATION = { + "view_args": ma.RAISE, + "path": ma.RAISE, + **core.Parser.DEFAULT_UNKNOWN_BY_LOCATION, + } __location_map__ = dict( view_args="load_view_args", path="load_view_args", diff --git a/src/webargs/pyramidparser.py b/src/webargs/pyramidparser.py index 063af6136..9537fb96e 100644 --- a/src/webargs/pyramidparser.py +++ b/src/webargs/pyramidparser.py @@ -30,6 +30,8 @@ def hello_world(request, args): from webob.multidict import MultiDict from pyramid.httpexceptions import exception_response +import marshmallow as ma + from webargs import core from webargs.core import json from webargs.multidictproxy import MultiDictProxy @@ -42,6 +44,11 @@ def is_json_request(req): class PyramidParser(core.Parser): """Pyramid request argument parser.""" + DEFAULT_UNKNOWN_BY_LOCATION = { + "matchdict": ma.RAISE, + "path": ma.RAISE, + **core.Parser.DEFAULT_UNKNOWN_BY_LOCATION, + } __location_map__ = dict( matchdict="load_matchdict", path="load_matchdict", diff --git a/tests/apps/aiohttp_app.py b/tests/apps/aiohttp_app.py index 8c1b24194..f7910a49b 100644 --- a/tests/apps/aiohttp_app.py +++ b/tests/apps/aiohttp_app.py @@ -75,7 +75,9 @@ async def echo_use_args_validated(request, args): async def echo_ignoring_extra_data(request): - return json_response(await parser.parse(hello_exclude_schema, request)) + return json_response( + await parser.parse(hello_exclude_schema, request, unknown=None) + ) async def echo_multi(request): @@ -124,7 +126,7 @@ def always_fail(value): async def echo_headers(request): - parsed = await parser.parse(hello_exclude_schema, request, location="headers") + parsed = await parser.parse(hello_args, request, location="headers") return json_response(parsed) diff --git a/tests/apps/bottle_app.py b/tests/apps/bottle_app.py index d6e3e9651..199cd9833 100644 --- a/tests/apps/bottle_app.py +++ b/tests/apps/bottle_app.py @@ -65,7 +65,7 @@ def echo_use_args_validated(args): @app.route("/echo_ignoring_extra_data", method=["POST"]) def echo_json_ignore_extra_data(): - return parser.parse(hello_exclude_schema) + return parser.parse(hello_exclude_schema, unknown=None) @app.route( @@ -123,9 +123,7 @@ def always_fail(value): @app.route("/echo_headers") def echo_headers(): - # the "exclude schema" must be used in this case because WSGI headers may - # be populated with many fields not sent by the caller - return parser.parse(hello_exclude_schema, request, location="headers") + return parser.parse(hello_args, request, location="headers") @app.route("/echo_cookie") diff --git a/tests/apps/django_app/echo/views.py b/tests/apps/django_app/echo/views.py index 88c01e09f..e37428f0a 100644 --- a/tests/apps/django_app/echo/views.py +++ b/tests/apps/django_app/echo/views.py @@ -73,7 +73,7 @@ def echo_use_args_validated(args): @handle_view_errors def echo_ignoring_extra_data(request): - return json_response(parser.parse(hello_exclude_schema, request)) + return json_response(parser.parse(hello_exclude_schema, request, unknown=None)) @handle_view_errors @@ -125,9 +125,7 @@ def always_fail(value): @handle_view_errors def echo_headers(request): - return json_response( - parser.parse(hello_exclude_schema, request, location="headers") - ) + return json_response(parser.parse(hello_args, request, location="headers")) @handle_view_errors diff --git a/tests/apps/falcon_app.py b/tests/apps/falcon_app.py index d3d786d6e..314a35a9b 100644 --- a/tests/apps/falcon_app.py +++ b/tests/apps/falcon_app.py @@ -67,7 +67,7 @@ def on_post(self, req, resp, args): class EchoJSONIgnoreExtraData: def on_post(self, req, resp): - resp.body = json.dumps(parser.parse(hello_exclude_schema, req)) + resp.body = json.dumps(parser.parse(hello_exclude_schema, req, unknown=None)) class EchoMulti: @@ -118,9 +118,7 @@ def on_get(self, req, resp): class HeaderSchema(ma.Schema): NAME = fields.Str(missing="World") - resp.body = json.dumps( - parser.parse(HeaderSchema(unknown=ma.EXCLUDE), req, location="headers") - ) + resp.body = json.dumps(parser.parse(HeaderSchema(), req, location="headers")) class EchoCookie: diff --git a/tests/apps/flask_app.py b/tests/apps/flask_app.py index 489e4e21e..68d01ee0e 100644 --- a/tests/apps/flask_app.py +++ b/tests/apps/flask_app.py @@ -21,9 +21,6 @@ class HelloSchema(ma.Schema): hello_many_schema = HelloSchema(many=True) -# variant which ignores unknown fields -hello_exclude_schema = HelloSchema(unknown=ma.EXCLUDE) - app = Flask(__name__) app.config.from_object(TestAppConfig) @@ -64,7 +61,7 @@ def echo_use_args_validated(args): @app.route("/echo_ignoring_extra_data", methods=["POST"]) def echo_json_ignore_extra_data(): - return J(parser.parse(hello_exclude_schema)) + return J(parser.parse(hello_args, unknown=ma.EXCLUDE)) @app.route("/echo_use_kwargs", methods=["GET"]) @@ -117,16 +114,14 @@ def always_fail(value): @app.route("/echo_headers") def echo_headers(): - # the "exclude schema" must be used in this case because WSGI headers may - # be populated with many fields not sent by the caller - return J(parser.parse(hello_exclude_schema, location="headers")) + return J(parser.parse(hello_args, location="headers")) +# as above, but in this case, turn off the default `EXCLUDE` behavior for +# `headers`, so that errors will be raised @app.route("/echo_headers_raising") -@use_args(HelloSchema(), location="headers") +@use_args(HelloSchema(), location="headers", unknown=None) def echo_headers_raising(args): - # as above, but in this case, don't use the exclude schema (so unexpected - # headers will raise errors) return J(args) diff --git a/tests/apps/pyramid_app.py b/tests/apps/pyramid_app.py index a1f4549e1..e22d26adb 100644 --- a/tests/apps/pyramid_app.py +++ b/tests/apps/pyramid_app.py @@ -50,7 +50,7 @@ def echo_json_or_form(request): def echo_json_ignore_extra_data(request): try: - return parser.parse(hello_exclude_schema, request) + return parser.parse(hello_exclude_schema, request, unknown=None) except json.JSONDecodeError: error = HTTPBadRequest() error.body = json.dumps(["Invalid JSON."]).encode("utf-8") @@ -114,7 +114,7 @@ def always_fail(value): def echo_headers(request): - return parser.parse(hello_exclude_schema, request, location="headers") + return parser.parse(hello_args, request, location="headers") def echo_cookie(request): diff --git a/tests/test_core.py b/tests/test_core.py index 83b17ab9a..ee76cbf59 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -109,7 +109,12 @@ def test_parse(parser, web_request): @pytest.mark.parametrize( "set_location", - ["schema_instance", "parse_call", "parser_default", "parser_class_default"], + [ + "schema_instance", + "parse_call", + "parser_default", + "parser_class_default", + ], ) def test_parse_with_unknown_behavior_specified(parser, web_request, set_location): web_request.json = {"username": 42, "password": 42, "fjords": 42} @@ -121,7 +126,11 @@ class CustomSchema(Schema): def parse_with_desired_behavior(value): if set_location == "schema_instance": if value is not None: - return parser.parse(CustomSchema(unknown=value), web_request) + # pass 'unknown=None' to parse() in order to indicate that the + # schema setting should be respected + return parser.parse( + CustomSchema(unknown=value), web_request, unknown=None + ) else: return parser.parse(CustomSchema(), web_request) elif set_location == "parse_call": @@ -132,7 +141,7 @@ def parse_with_desired_behavior(value): elif set_location == "parser_class_default": class CustomParser(MockRequestParser): - DEFAULT_UNKNOWN = value + DEFAULT_UNKNOWN_BY_LOCATION = {"json": value} return CustomParser().parse(CustomSchema(), web_request) else: @@ -173,6 +182,44 @@ class CustomSchema(Schema): assert {"username": 42, "password": 42, "fjords": 42} == ret +@pytest.mark.parametrize("clear_method", ["custom_class", "instance_setting", "both"]) +def test_parse_with_default_unknown_cleared_uses_schema_value( + parser, web_request, clear_method +): + web_request.json = {"username": 42, "password": 42, "fjords": 42} + + class CustomSchema(Schema): + username = fields.Field() + password = fields.Field() + + if clear_method == "custom_class": + + class CustomParser(MockRequestParser): + DEFAULT_UNKNOWN_BY_LOCATION = {} + + parser = CustomParser() + elif clear_method == "instance_setting": + parser = MockRequestParser(unknown=None) + elif clear_method == "both": + # setting things in multiple ways should not result in errors + class CustomParser(MockRequestParser): + DEFAULT_UNKNOWN_BY_LOCATION = {} + + parser = CustomParser(unknown=None) + else: + raise NotImplementedError + + with pytest.raises(ValidationError, match="Unknown field."): + parser.parse(CustomSchema(), web_request) + with pytest.raises(ValidationError, match="Unknown field."): + parser.parse(CustomSchema(unknown=RAISE), web_request) + + ret = parser.parse(CustomSchema(unknown=EXCLUDE), web_request) + assert {"username": 42, "password": 42} == ret + ret = parser.parse(CustomSchema(unknown=INCLUDE), web_request) + assert {"username": 42, "password": 42, "fjords": 42} == ret + + def test_parse_required_arg_raises_validation_error(parser, web_request): web_request.json = {} args = {"foo": fields.Field(required=True)}