diff --git a/httplint/field/parsers/accept.py b/httplint/field/parsers/accept.py index 6bebe09..fd2c048 100644 --- a/httplint/field/parsers/accept.py +++ b/httplint/field/parsers/accept.py @@ -66,7 +66,7 @@ def evaluate(self, add_note: AddNoteMethodType) -> None: class BAD_Q_VALUE(Note): category = categories.CONNEG level = levels.WARN - _summary = "The q value on '{media_type}' is invalid." + _summary = "The q value on '%(media_type)s' is invalid." _text = """\ The `q` parameter must be a decimal number between 0 and 1, with at most 3 digits of precision.""" diff --git a/httplint/field/parsers/accept_patch.py b/httplint/field/parsers/accept_patch.py index 2e3c08f..247f2a0 100644 --- a/httplint/field/parsers/accept_patch.py +++ b/httplint/field/parsers/accept_patch.py @@ -1,9 +1,11 @@ from typing import Tuple +from httplint.field import BAD_SYNTAX from httplint.field.list_field import HttpListField from httplint.field.tests import FieldTest from httplint.field.utils import parse_media_type from httplint.note import Note, categories, levels +from httplint.syntax import rfc9110 from httplint.types import ( AddNoteMethodType, NoteClassListType, @@ -18,16 +20,12 @@ class accept_patch(HttpListField[ResponseLinterProtocol]): The `Accept-Patch` response header advertises which media types are accepted by the server in a PATCH request.""" reference = "https://www.rfc-editor.org/rfc/rfc5789.html#section-3.1" - syntax = False + syntax = rfc9110.list_rule(rfc9110.media_type, 1) category = categories.GENERAL deprecated = False - def parse( - self, field_value: str, add_note: AddNoteMethodType - ) -> Tuple[str, ParamDictType]: - return parse_media_type( - field_value, add_note, ACCEPT_PATCH_BAD_SYNTAX, self.reference - ) + def parse(self, field_value: str, add_note: AddNoteMethodType) -> Tuple[str, ParamDictType]: + return parse_media_type(field_value, add_note, ACCEPT_PATCH_BAD_SYNTAX, self.reference) class ACCEPT_PATCH_BAD_SYNTAX(Note): @@ -59,4 +57,13 @@ class AcceptPatchBadTest(FieldTest[ResponseLinterProtocol]): name = "Accept-Patch" inputs = [b"invalid"] expected_out = [("invalid", {})] + expected_notes: NoteClassListType = [ACCEPT_PATCH_BAD_SYNTAX, BAD_SYNTAX] + + +class AcceptPatchWildcardTest(FieldTest[ResponseLinterProtocol]): + "Accept-Patch lists media types, not media ranges." + + name = "Accept-Patch" + inputs = [b"*/*"] + expected_out = [("*/*", {})] expected_notes: NoteClassListType = [ACCEPT_PATCH_BAD_SYNTAX] diff --git a/httplint/field/parsers/accept_post.py b/httplint/field/parsers/accept_post.py index f7b841b..751d9c4 100644 --- a/httplint/field/parsers/accept_post.py +++ b/httplint/field/parsers/accept_post.py @@ -1,9 +1,11 @@ from typing import Tuple +from httplint.field import BAD_SYNTAX from httplint.field.list_field import HttpListField from httplint.field.tests import FieldTest from httplint.field.utils import parse_media_type from httplint.note import Note, categories, levels +from httplint.syntax import rfc9110 from httplint.types import ( AddNoteMethodType, NoteClassListType, @@ -18,13 +20,12 @@ class accept_post(HttpListField[ResponseLinterProtocol]): The `Accept-Post` response header advertises which media types are accepted by the server in a POST request.""" reference = "https://www.w3.org/TR/ldp/#header-accept-post" - syntax = False + # LDP defines this as #media-range, not #media-type; wildcards are allowed. + syntax = rfc9110.list_rule(rfc9110.media_range) category = categories.GENERAL deprecated = False - def parse( - self, field_value: str, add_note: AddNoteMethodType - ) -> Tuple[str, ParamDictType]: + def parse(self, field_value: str, add_note: AddNoteMethodType) -> Tuple[str, ParamDictType]: return parse_media_type( field_value, add_note, @@ -60,4 +61,4 @@ class AcceptPostBadTest(FieldTest[ResponseLinterProtocol]): name = "Accept-Post" inputs = [b"invalid"] expected_out = [("invalid", {})] - expected_notes: NoteClassListType = [ACCEPT_POST_BAD_SYNTAX] + expected_notes: NoteClassListType = [ACCEPT_POST_BAD_SYNTAX, BAD_SYNTAX] diff --git a/httplint/field/parsers/accept_query.py b/httplint/field/parsers/accept_query.py index 4078e5b..16c836a 100644 --- a/httplint/field/parsers/accept_query.py +++ b/httplint/field/parsers/accept_query.py @@ -1,46 +1,75 @@ -from typing import Tuple +from http_sf import Token -from httplint.field.list_field import HttpListField +from httplint.field.structured_field import StructuredField from httplint.field.tests import FieldTest -from httplint.field.utils import parse_media_type +from httplint.field.utils import check_media_type from httplint.note import Note, categories, levels from httplint.types import ( AddNoteMethodType, NoteClassListType, - ParamDictType, ResponseLinterProtocol, + SFListType, ) +SPEC_URL = "https://www.rfc-editor.org/rfc/rfc10008.html" -class accept_query(HttpListField[ResponseLinterProtocol]): + +class accept_query(StructuredField[ResponseLinterProtocol]): canonical_name = "Accept-Query" description = """\ The `Accept-Query` response header advertises which media types are accepted by the server in the content of a QUERY request.""" - reference = ( - "https://datatracker.ietf.org/doc/html/" - "draft-ietf-httpbis-safe-method-w-body#section-3" - ) - syntax = False + reference = f"{SPEC_URL}#section-3" + syntax = False # Structured Field category = categories.GENERAL deprecated = False + sf_type = "list" + value: SFListType + + def evaluate(self, add_note: AddNoteMethodType) -> None: + normalised: SFListType = [] + for item in self.value: + # SF List items are (value, parameters) tuples + val, params = item + if not isinstance(val, (Token, str)): + add_note(ACCEPT_QUERY_BAD_TYPE, value=str(val)) + normalised.append(item) + continue + # Media type parameters are carried as SF parameters, so the item + # value is the media range on its own. Media ranges are + # case-insensitive, so store them lowercased, as the other + # media-type fields do. + media_range = str(val).lower() + check_media_type( + media_range, + add_note, + ACCEPT_QUERY_BAD_SYNTAX, + self.reference, + allow_wildcard=True, + check_token=True, # no syntax check on a Structured Field + ) + normalised.append((type(val)(media_range), params)) + self.value = normalised - def parse( - self, field_value: str, add_note: AddNoteMethodType - ) -> Tuple[str, ParamDictType]: - return parse_media_type( - field_value, add_note, ACCEPT_QUERY_BAD_SYNTAX, self.reference - ) + +class ACCEPT_QUERY_BAD_TYPE(Note): + category = categories.GENERAL + level = levels.BAD + _summary = "The Accept-Query header contains a value that isn't a media range." + _text = """\ +`Accept-Query` is a List Structured Field whose members are Tokens or Strings, each +naming a media range accepted in the content of a QUERY request. `%(value)s` is +neither, so it will be ignored.""" class ACCEPT_QUERY_BAD_SYNTAX(Note): category = categories.GENERAL level = levels.BAD - _summary = "The Accept-Query header contains a value that is not a media type." + _summary = "The Accept-Query header contains a value that is not a media range." _text = """\ -`%(value)s` is not a valid media type. `Accept-Query` is a list of media types -(e.g., `application/sparql-query`) accepted in the content of a QUERY request; -see [its definition](%(ref_uri)s) for more information.""" +`%(value)s` is not a valid media range. `Accept-Query` is a list of media ranges +(e.g., `application/sparql-query`, `text/*`) accepted in the content of a QUERY +request; see [its definition](%(ref_uri)s) for more information.""" class AcceptQueryTest(FieldTest[ResponseLinterProtocol]): @@ -49,10 +78,30 @@ class AcceptQueryTest(FieldTest[ResponseLinterProtocol]): expected_out = [("application/sparql-query", {}), ("application/sql", {})] +class AcceptQueryStringTest(FieldTest[ResponseLinterProtocol]): + "Media types that aren't valid Tokens have to be sent as Strings." + + name = "Accept-Query" + inputs = [b'"application/jsonpath", "3d/example"'] + expected_out = [("application/jsonpath", {}), ("3d/example", {})] + + class AcceptQueryParamsTest(FieldTest[ResponseLinterProtocol]): name = "Accept-Query" - inputs = [b"application/example;version=1"] - expected_out = [("application/example", {"version": "1"})] + inputs = [b'application/sql;charset="UTF-8"'] + expected_out = [("application/sql", {"charset": "UTF-8"})] + + +class AcceptQueryWildcardTest(FieldTest[ResponseLinterProtocol]): + name = "Accept-Query" + inputs = [b"*/*, text/*"] + expected_out = [("*/*", {}), ("text/*", {})] + + +class AcceptQueryCaseTest(FieldTest[ResponseLinterProtocol]): + name = "Accept-Query" + inputs = [b'APPLICATION/SQL, "TEXT/Plain"'] + expected_out = [("application/sql", {}), ("text/plain", {})] class AcceptQueryBadTest(FieldTest[ResponseLinterProtocol]): @@ -60,3 +109,28 @@ class AcceptQueryBadTest(FieldTest[ResponseLinterProtocol]): inputs = [b"invalid"] expected_out = [("invalid", {})] expected_notes: NoteClassListType = [ACCEPT_QUERY_BAD_SYNTAX] + + +class AcceptQueryBadStringTest(FieldTest[ResponseLinterProtocol]): + "A String member can carry a name that isn't a valid HTTP token." + + name = "Accept-Query" + inputs = [b'"text/pl in"'] + expected_out = [("text/pl in", {})] + expected_notes: NoteClassListType = [ACCEPT_QUERY_BAD_SYNTAX] + + +class AcceptQueryBareStarTest(FieldTest[ResponseLinterProtocol]): + "Only */* and type/* are permitted, not a bare *." + + name = "Accept-Query" + inputs = [b"*"] + expected_out = [("*", {})] + expected_notes: NoteClassListType = [ACCEPT_QUERY_BAD_SYNTAX] + + +class AcceptQueryBadTypeTest(FieldTest[ResponseLinterProtocol]): + name = "Accept-Query" + inputs = [b"123"] + expected_out = [(123, {})] + expected_notes: NoteClassListType = [ACCEPT_QUERY_BAD_TYPE] diff --git a/httplint/field/parsers/content_type.py b/httplint/field/parsers/content_type.py index e06258c..bdf572f 100644 --- a/httplint/field/parsers/content_type.py +++ b/httplint/field/parsers/content_type.py @@ -1,12 +1,17 @@ -from typing import Tuple +from typing import Any, Tuple from httplint.field.singleton_field import SingletonField from httplint.field.tests import FieldTest -from httplint.field.utils import parse_media_type +from httplint.field.utils import ( + MEDIA_TYPE_BAD_NAME, + MEDIA_TYPE_LONG_NAME, + parse_media_type, +) from httplint.syntax import rfc9110 from httplint.types import ( AddNoteMethodType, AnyMessageLinterProtocol, + NoteClassListType, ParamDictType, ) @@ -29,3 +34,43 @@ class BasicCTTest(FieldTest[AnyMessageLinterProtocol]): name = "Content-Type" inputs = [b"text/plain; charset=utf-8"] expected_out = ("text/plain", {"charset": "utf-8"}) + + +class CTSuffixTest(FieldTest[AnyMessageLinterProtocol]): + name = "Content-Type" + inputs = [b"application/vnd.example.foo-bar+json"] + expected_out: Any = ("application/vnd.example.foo-bar+json", {}) + + +class CTBadNameTest(FieldTest[AnyMessageLinterProtocol]): + "A media type that's a valid HTTP token, but not a valid RFC 6838 name." + + name = "Content-Type" + inputs = [b"text/pl~in"] + expected_out: Any = ("text/pl~in", {}) + expected_notes: NoteClassListType = [MEDIA_TYPE_BAD_NAME] + + +class CTBadTypeNameTest(FieldTest[AnyMessageLinterProtocol]): + "The type half is checked as well as the subtype half." + + name = "Content-Type" + inputs = [b"~text/plain"] + expected_out: Any = ("~text/plain", {}) + expected_notes: NoteClassListType = [MEDIA_TYPE_BAD_NAME] + + +class CTBadNameFirstTest(FieldTest[AnyMessageLinterProtocol]): + "RFC 6838 names have to start with a letter or a digit." + + name = "Content-Type" + inputs = [b"text/.plain"] + expected_out: Any = ("text/.plain", {}) + expected_notes: NoteClassListType = [MEDIA_TYPE_BAD_NAME] + + +class CTLongNameTest(FieldTest[AnyMessageLinterProtocol]): + name = "Content-Type" + inputs = [b"text/" + b"a" * 128] + expected_out: Any = ("text/" + "a" * 128, {}) + expected_notes: NoteClassListType = [MEDIA_TYPE_LONG_NAME] diff --git a/httplint/field/utils.py b/httplint/field/utils.py index 3697f69..2098d58 100644 --- a/httplint/field/utils.py +++ b/httplint/field/utils.py @@ -7,9 +7,19 @@ from http_sf import Token from httplint.note import Note, categories, levels -from httplint.syntax import rfc9110 +from httplint.syntax import rfc6838, rfc9110 from httplint.types import AddNoteMethodType, ParamDictType +RE_FLAGS = re.VERBOSE | re.IGNORECASE + +# restricted-name (RFC 6838, Section 4.2) without its 126-character bound, so that +# over-long names are reported as MEDIA_TYPE_LONG_NAME instead of being lumped in +# with character errors. + +RESTRICTED_NAME_UNBOUNDED = ( + rf"(?: {rfc6838.restricted_name_first} {rfc6838.restricted_name_chars}* )" +) + def parse_media_type( field_value: str, @@ -22,27 +32,77 @@ def parse_media_type( """ Parse a media-type with optional parameters (e.g. ``text/html;charset=utf-8``). - Returns a tuple of the lowercased media-type and its parameter dict. If - ``bad_syntax_note`` is provided, it is emitted (with ``ref_uri`` if given) - when the media-type lacks a ``/``. When ``allow_wildcard`` is true, a bare - ``*`` is accepted without a note. ``nostar`` is passed through to - ``parse_params`` to flag RFC 5987-style ``param*`` keys. + Returns a tuple of the lowercased media-type and its parameter dict; see + ``check_media_type`` for the checks applied along the way. ``nostar`` is passed + through to ``parse_params`` to flag RFC 5987-style ``param*`` keys. """ try: media_type, param_str = field_value.split(";", 1) except ValueError: media_type, param_str = field_value, "" media_type = media_type.strip().lower() - if "/" not in media_type and not (allow_wildcard and media_type == "*"): - if bad_syntax_note is not None: - kwargs: Dict[str, Any] = {"value": media_type} - if ref_uri is not None: - kwargs["ref_uri"] = ref_uri - add_note(bad_syntax_note, **kwargs) + check_media_type(media_type, add_note, bad_syntax_note, ref_uri, allow_wildcard) param_dict = parse_params(param_str, add_note, nostar) return media_type, param_dict -RE_FLAGS = re.VERBOSE | re.IGNORECASE + +def check_media_type( + media_type: str, + add_note: AddNoteMethodType, + bad_syntax_note: Optional[Type[Note]] = None, + ref_uri: Optional[str] = None, + allow_wildcard: bool = False, + check_token: bool = False, +) -> None: + """ + Check a media-type against the shape HTTP gives it and the naming rules in + RFC 6838, Section 4.2. + + If ``bad_syntax_note`` is provided, it is emitted (with ``ref_uri`` if given) + when the value isn't shaped like a media-type at all. When ``allow_wildcard`` + is true the value is treated as a media range, so ``*/*`` and ``type/*`` are + accepted; otherwise a ``*`` in either position is an error. + + Names that aren't valid HTTP tokens are normally left to the field's own + syntax check. Callers without one -- Structured Fields, whose members can + carry any string -- should set ``check_token`` so that they're reported here + instead. + """ + + def bad_syntax() -> None: + if bad_syntax_note is None: + return + kwargs: Dict[str, Any] = {"value": media_type} + if ref_uri is not None: + kwargs["ref_uri"] = ref_uri + add_note(bad_syntax_note, **kwargs) + + type_name, slash, subtype_name = media_type.partition("/") + if not (slash and type_name and subtype_name) or "/" in subtype_name: + bad_syntax() + return + + names = [type_name, subtype_name] + if "*" in names: + # "*/*" and "type/*" are media ranges; "*/subtype" isn't anything. + if not allow_wildcard or (type_name == "*" and subtype_name != "*"): + bad_syntax() + return + names = [name for name in names if name != "*"] + + tokens = [name for name in names if re.match(rf"^{rfc9110.token}$", name, RE_FLAGS)] + if len(tokens) != len(names): + if check_token: + bad_syntax() + return + # Otherwise the field's own syntax check reports it, and RFC 6838 has + # nothing to add. + names = tokens + + if any(len(name) > rfc6838.RESTRICTED_NAME_MAX_LEN for name in names): + add_note(MEDIA_TYPE_LONG_NAME, value=media_type) + if any(not re.match(rf"^{RESTRICTED_NAME_UNBOUNDED}$", name, RE_FLAGS) for name in names): + add_note(MEDIA_TYPE_BAD_NAME, value=media_type) def parse_http_date( @@ -101,14 +161,17 @@ def split_string(instr: str, item: str, split: str) -> List[str]: def split_list_field(field_value: str) -> List[str]: "Split a field field value on commas. needs to conform to the #rule." return [ - f.strip() - for f in re.findall( - r'((?:[^",]|%s)+)(?=%s|\s*$)' % (rfc9110.quoted_string, r"(?:\s*(?:,\s*)+)"), - field_value, - RE_FLAGS, + stripped + for stripped in ( + f.strip() + for f in re.findall( + r'((?:[^",]|%s)+)(?=%s|\s*$)' % (rfc9110.quoted_string, r"(?:\s*(?:,\s*)+)"), + field_value, + RE_FLAGS, + ) ) - if f - ] or [] + if stripped + ] def parse_params( @@ -244,6 +307,32 @@ def check_sf_item_token( add_note(invalid_note, value=field_value, **kwargs) +class MEDIA_TYPE_BAD_NAME(Note): + category = categories.GENERAL + level = levels.WARN + _summary = "The %(field_name)s field's media type name isn't registrable." + _text = """\ +[RFC6838](https://www.rfc-editor.org/rfc/rfc6838.html#section-4.2) requires the type and +subtype names in a media type to start with a letter or a digit, and to use only letters, +digits and the characters `!`, `#`, `$`, `&`, `-`, `^`, `_`, `.` and `+`. `%(value)s` +uses something else. + +HTTP's own syntax for media types is more permissive, so recipients are likely to accept +this; however, a media type named this way can't be registered with IANA.""" + + +class MEDIA_TYPE_LONG_NAME(Note): + category = categories.GENERAL + level = levels.WARN + _summary = "The %(field_name)s field's media type name is too long." + _text = """\ +[RFC6838](https://www.rfc-editor.org/rfc/rfc6838.html#section-4.2) limits the type and +subtype names in a media type to 127 characters each; `%(value)s` is longer than that. + +It also recommends keeping them to 64 characters, because longer names run into +implementation limits.""" + + class PARAM_REPEATS(Note): category = categories.GENERAL level = levels.WARN diff --git a/httplint/syntax/__init__.py b/httplint/syntax/__init__.py index d64a1fe..f2f04a4 100755 --- a/httplint/syntax/__init__.py +++ b/httplint/syntax/__init__.py @@ -10,6 +10,7 @@ "rfc5322", "rfc5646", "rfc5987", + "rfc6838", "rfc8288", "rfc9110", "rfc9111", diff --git a/httplint/syntax/rfc6838.py b/httplint/syntax/rfc6838.py new file mode 100644 index 0000000..1c6b3a6 --- /dev/null +++ b/httplint/syntax/rfc6838.py @@ -0,0 +1,42 @@ +""" +Regex for RFC6838 +""" + +# pylint: disable=invalid-name + +from .rfc5234 import ( + ALPHA, + DIGIT, +) + +SPEC_URL = "https://www.rfc-editor.org/rfc/rfc6838" + + +# restricted-name-first = ALPHA / DIGIT + +restricted_name_first = rf"(?: {ALPHA} | {DIGIT} )" + +# restricted-name-chars = ALPHA / DIGIT / "!" / "#" / +# "$" / "&" / "-" / "^" / "_" +# restricted-name-chars =/ "." ; Characters before first dot always +# ; specify a facet name +# restricted-name-chars =/ "+" ; Characters after last plus always +# ; specify a structured syntax suffix + +restricted_name_chars = rf"(?: {ALPHA} | {DIGIT} | ! | \# | \$ | & | \- | \^ | _ | \. | \+ )" + +# restricted-name = restricted-name-first *126restricted-name-chars + +restricted_name = rf"(?: {restricted_name_first} {restricted_name_chars}{{0,126}} )" + +# type-name = restricted-name + +type_name = restricted_name + +# subtype-name = restricted-name + +subtype_name = restricted_name + +# The longest a type-name or subtype-name can be, per restricted-name above. + +RESTRICTED_NAME_MAX_LEN = 127