From b886ffb0111c892c44dbd6bcc3d1466f49a03b97 Mon Sep 17 00:00:00 2001 From: Mark Nottingham Date: Sat, 8 Aug 2026 15:26:09 +1000 Subject: [PATCH 1/4] Added: check media type names, and the syntax of Accept-Patch/Accept-Post parse_media_type() only checked that a media type contained a "/", leaving everything else to each field's ABNF regex. Accept-Patch, Accept-Post and Accept-Query had syntax = False, so they got no character-level checking at all: "@@@/###", '"quoted"/thing', "text/" and "a/b/c" all passed silently. Give Accept-Patch and Accept-Post the syntax their specs define (RFC 5789 is 1#media-type; LDP is #media-range, so it keeps wildcards), and move the media-type checks proper into check_media_type(): * The shape check now requires exactly one "/" with a non-empty name on each side, rather than just a "/" somewhere. * allow_wildcard now means something. It only exempted a bare "*" before, so "*/*" and "type/*" were accepted everywhere -- including on Accept-Patch, which lists media types, not media ranges. It now gates "*" in either position, and "*/subtype" is rejected in all cases. * Type and subtype names are checked against restricted-name from RFC 6838, Section 4.2, added as httplint/syntax/rfc6838.py. HTTP's token production is more permissive than the registry's rules, so "~text/pl%in" was clean before and is now MEDIA_TYPE_BAD_NAME; names over 127 characters are MEDIA_TYPE_LONG_NAME. Both are WARN, since such a value is legal HTTP and recipients will accept it -- it just can't be registered with IANA. Names that aren't valid tokens are left to the field's own syntax check, so there's no double reporting. Accept-Query keeps syntax = False; the current draft defines it as a Structured Field, so the media-type ABNF would reject its valid forms. Also fix split_list_field(), which tested each match for truth before stripping it, so a whitespace-only element survived as "". That produced a spurious bad-syntax note for values with trailing whitespace after a comma, e.g. "Accept-Patch: text/plain, ". Co-Authored-By: Claude Opus 5 --- httplint/field/parsers/accept_patch.py | 21 +++-- httplint/field/parsers/accept_post.py | 11 +-- httplint/field/parsers/accept_query.py | 17 ++-- httplint/field/parsers/content_type.py | 40 ++++++++- httplint/field/utils.py | 119 ++++++++++++++++++++----- httplint/syntax/__init__.py | 1 + httplint/syntax/rfc6838.py | 42 +++++++++ 7 files changed, 206 insertions(+), 45 deletions(-) create mode 100644 httplint/syntax/rfc6838.py diff --git a/httplint/field/parsers/accept_patch.py b/httplint/field/parsers/accept_patch.py index 2e3c08f..247f2a0 100644 --- a/httplint/field/parsers/accept_patch.py +++ b/httplint/field/parsers/accept_patch.py @@ -1,9 +1,11 @@ from typing import Tuple +from httplint.field import BAD_SYNTAX from httplint.field.list_field import HttpListField from httplint.field.tests import FieldTest from httplint.field.utils import parse_media_type from httplint.note import Note, categories, levels +from httplint.syntax import rfc9110 from httplint.types import ( AddNoteMethodType, NoteClassListType, @@ -18,16 +20,12 @@ class accept_patch(HttpListField[ResponseLinterProtocol]): The `Accept-Patch` response header advertises which media types are accepted by the server in a PATCH request.""" reference = "https://www.rfc-editor.org/rfc/rfc5789.html#section-3.1" - syntax = False + syntax = rfc9110.list_rule(rfc9110.media_type, 1) category = categories.GENERAL deprecated = False - def parse( - self, field_value: str, add_note: AddNoteMethodType - ) -> Tuple[str, ParamDictType]: - return parse_media_type( - field_value, add_note, ACCEPT_PATCH_BAD_SYNTAX, self.reference - ) + def parse(self, field_value: str, add_note: AddNoteMethodType) -> Tuple[str, ParamDictType]: + return parse_media_type(field_value, add_note, ACCEPT_PATCH_BAD_SYNTAX, self.reference) class ACCEPT_PATCH_BAD_SYNTAX(Note): @@ -59,4 +57,13 @@ class AcceptPatchBadTest(FieldTest[ResponseLinterProtocol]): name = "Accept-Patch" inputs = [b"invalid"] expected_out = [("invalid", {})] + expected_notes: NoteClassListType = [ACCEPT_PATCH_BAD_SYNTAX, BAD_SYNTAX] + + +class AcceptPatchWildcardTest(FieldTest[ResponseLinterProtocol]): + "Accept-Patch lists media types, not media ranges." + + name = "Accept-Patch" + inputs = [b"*/*"] + expected_out = [("*/*", {})] expected_notes: NoteClassListType = [ACCEPT_PATCH_BAD_SYNTAX] diff --git a/httplint/field/parsers/accept_post.py b/httplint/field/parsers/accept_post.py index f7b841b..751d9c4 100644 --- a/httplint/field/parsers/accept_post.py +++ b/httplint/field/parsers/accept_post.py @@ -1,9 +1,11 @@ from typing import Tuple +from httplint.field import BAD_SYNTAX from httplint.field.list_field import HttpListField from httplint.field.tests import FieldTest from httplint.field.utils import parse_media_type from httplint.note import Note, categories, levels +from httplint.syntax import rfc9110 from httplint.types import ( AddNoteMethodType, NoteClassListType, @@ -18,13 +20,12 @@ class accept_post(HttpListField[ResponseLinterProtocol]): The `Accept-Post` response header advertises which media types are accepted by the server in a POST request.""" reference = "https://www.w3.org/TR/ldp/#header-accept-post" - syntax = False + # LDP defines this as #media-range, not #media-type; wildcards are allowed. + syntax = rfc9110.list_rule(rfc9110.media_range) category = categories.GENERAL deprecated = False - def parse( - self, field_value: str, add_note: AddNoteMethodType - ) -> Tuple[str, ParamDictType]: + def parse(self, field_value: str, add_note: AddNoteMethodType) -> Tuple[str, ParamDictType]: return parse_media_type( field_value, add_note, @@ -60,4 +61,4 @@ class AcceptPostBadTest(FieldTest[ResponseLinterProtocol]): name = "Accept-Post" inputs = [b"invalid"] expected_out = [("invalid", {})] - expected_notes: NoteClassListType = [ACCEPT_POST_BAD_SYNTAX] + expected_notes: NoteClassListType = [ACCEPT_POST_BAD_SYNTAX, BAD_SYNTAX] diff --git a/httplint/field/parsers/accept_query.py b/httplint/field/parsers/accept_query.py index 4078e5b..9fc8d1e 100644 --- a/httplint/field/parsers/accept_query.py +++ b/httplint/field/parsers/accept_query.py @@ -11,26 +11,21 @@ ResponseLinterProtocol, ) +SPEC_URL = "https://datatracker.ietf.org/doc/html/draft-ietf-httpbis-safe-method-w-body" + class accept_query(HttpListField[ResponseLinterProtocol]): canonical_name = "Accept-Query" description = """\ The `Accept-Query` response header advertises which media types are accepted by the server in the content of a QUERY request.""" - reference = ( - "https://datatracker.ietf.org/doc/html/" - "draft-ietf-httpbis-safe-method-w-body#section-3" - ) - syntax = False + reference = f"{SPEC_URL}#section-3" + syntax = False # Structured Field category = categories.GENERAL deprecated = False - def parse( - self, field_value: str, add_note: AddNoteMethodType - ) -> Tuple[str, ParamDictType]: - return parse_media_type( - field_value, add_note, ACCEPT_QUERY_BAD_SYNTAX, self.reference - ) + def parse(self, field_value: str, add_note: AddNoteMethodType) -> Tuple[str, ParamDictType]: + return parse_media_type(field_value, add_note, ACCEPT_QUERY_BAD_SYNTAX, self.reference) class ACCEPT_QUERY_BAD_SYNTAX(Note): diff --git a/httplint/field/parsers/content_type.py b/httplint/field/parsers/content_type.py index e06258c..f116e8a 100644 --- a/httplint/field/parsers/content_type.py +++ b/httplint/field/parsers/content_type.py @@ -1,12 +1,17 @@ -from typing import Tuple +from typing import Any, Tuple from httplint.field.singleton_field import SingletonField from httplint.field.tests import FieldTest -from httplint.field.utils import parse_media_type +from httplint.field.utils import ( + MEDIA_TYPE_BAD_NAME, + MEDIA_TYPE_LONG_NAME, + parse_media_type, +) from httplint.syntax import rfc9110 from httplint.types import ( AddNoteMethodType, AnyMessageLinterProtocol, + NoteClassListType, ParamDictType, ) @@ -29,3 +34,34 @@ class BasicCTTest(FieldTest[AnyMessageLinterProtocol]): name = "Content-Type" inputs = [b"text/plain; charset=utf-8"] expected_out = ("text/plain", {"charset": "utf-8"}) + + +class CTSuffixTest(FieldTest[AnyMessageLinterProtocol]): + name = "Content-Type" + inputs = [b"application/vnd.example.foo-bar+json"] + expected_out: Any = ("application/vnd.example.foo-bar+json", {}) + + +class CTBadNameTest(FieldTest[AnyMessageLinterProtocol]): + "A media type that's a valid HTTP token, but not a valid RFC 6838 name." + + name = "Content-Type" + inputs = [b"text/pl~in"] + expected_out: Any = ("text/pl~in", {}) + expected_notes: NoteClassListType = [MEDIA_TYPE_BAD_NAME] + + +class CTBadNameFirstTest(FieldTest[AnyMessageLinterProtocol]): + "RFC 6838 names have to start with a letter or a digit." + + name = "Content-Type" + inputs = [b"text/.plain"] + expected_out: Any = ("text/.plain", {}) + expected_notes: NoteClassListType = [MEDIA_TYPE_BAD_NAME] + + +class CTLongNameTest(FieldTest[AnyMessageLinterProtocol]): + name = "Content-Type" + inputs = [b"text/" + b"a" * 128] + expected_out: Any = ("text/" + "a" * 128, {}) + expected_notes: NoteClassListType = [MEDIA_TYPE_LONG_NAME] diff --git a/httplint/field/utils.py b/httplint/field/utils.py index 3697f69..d2e2e8c 100644 --- a/httplint/field/utils.py +++ b/httplint/field/utils.py @@ -7,9 +7,17 @@ from http_sf import Token from httplint.note import Note, categories, levels -from httplint.syntax import rfc9110 +from httplint.syntax import rfc6838, rfc9110 from httplint.types import AddNoteMethodType, ParamDictType +RE_FLAGS = re.VERBOSE | re.IGNORECASE + +# restricted-name (RFC 6838, Section 4.2) without its 126-character bound, so that +# over-long names are reported as MEDIA_TYPE_LONG_NAME instead of being lumped in +# with character errors. + +RESTRICTED_NAME_CHARS = rf"(?: {rfc6838.restricted_name_first} {rfc6838.restricted_name_chars}* )" + def parse_media_type( field_value: str, @@ -22,27 +30,69 @@ def parse_media_type( """ Parse a media-type with optional parameters (e.g. ``text/html;charset=utf-8``). - Returns a tuple of the lowercased media-type and its parameter dict. If - ``bad_syntax_note`` is provided, it is emitted (with ``ref_uri`` if given) - when the media-type lacks a ``/``. When ``allow_wildcard`` is true, a bare - ``*`` is accepted without a note. ``nostar`` is passed through to - ``parse_params`` to flag RFC 5987-style ``param*`` keys. + Returns a tuple of the lowercased media-type and its parameter dict; see + ``check_media_type`` for the checks applied along the way. ``nostar`` is passed + through to ``parse_params`` to flag RFC 5987-style ``param*`` keys. """ try: media_type, param_str = field_value.split(";", 1) except ValueError: media_type, param_str = field_value, "" media_type = media_type.strip().lower() - if "/" not in media_type and not (allow_wildcard and media_type == "*"): - if bad_syntax_note is not None: - kwargs: Dict[str, Any] = {"value": media_type} - if ref_uri is not None: - kwargs["ref_uri"] = ref_uri - add_note(bad_syntax_note, **kwargs) + check_media_type(media_type, add_note, bad_syntax_note, ref_uri, allow_wildcard) param_dict = parse_params(param_str, add_note, nostar) return media_type, param_dict -RE_FLAGS = re.VERBOSE | re.IGNORECASE + +def check_media_type( + media_type: str, + add_note: AddNoteMethodType, + bad_syntax_note: Optional[Type[Note]] = None, + ref_uri: Optional[str] = None, + allow_wildcard: bool = False, +) -> None: + """ + Check a media-type against the shape HTTP gives it and the naming rules in + RFC 6838, Section 4.2. + + If ``bad_syntax_note`` is provided, it is emitted (with ``ref_uri`` if given) + when the value isn't shaped like a media-type at all. When ``allow_wildcard`` + is true the value is treated as a media range, so ``*``, ``*/*`` and + ``type/*`` are accepted; otherwise a ``*`` in either position is an error. + """ + + def bad_syntax() -> None: + if bad_syntax_note is None: + return + kwargs: Dict[str, Any] = {"value": media_type} + if ref_uri is not None: + kwargs["ref_uri"] = ref_uri + add_note(bad_syntax_note, **kwargs) + + if allow_wildcard and media_type == "*": + return + + type_name, slash, subtype_name = media_type.partition("/") + if not (slash and type_name and subtype_name) or "/" in subtype_name: + bad_syntax() + return + + names = [type_name, subtype_name] + if "*" in names: + # "*/*" and "type/*" are media ranges; "*/subtype" isn't anything. + if not allow_wildcard or (type_name == "*" and subtype_name != "*"): + bad_syntax() + return + names = [name for name in names if name != "*"] + + # Names that aren't valid HTTP tokens are already reported by the field's own + # syntax check; RFC 6838 only adds information for the ones that are. + names = [name for name in names if re.match(rf"^{rfc9110.token}$", name, RE_FLAGS)] + + if any(len(name) > rfc6838.RESTRICTED_NAME_MAX_LEN for name in names): + add_note(MEDIA_TYPE_LONG_NAME, value=media_type) + if any(not re.match(rf"^{RESTRICTED_NAME_CHARS}$", name, RE_FLAGS) for name in names): + add_note(MEDIA_TYPE_BAD_NAME, value=media_type) def parse_http_date( @@ -101,14 +151,17 @@ def split_string(instr: str, item: str, split: str) -> List[str]: def split_list_field(field_value: str) -> List[str]: "Split a field field value on commas. needs to conform to the #rule." return [ - f.strip() - for f in re.findall( - r'((?:[^",]|%s)+)(?=%s|\s*$)' % (rfc9110.quoted_string, r"(?:\s*(?:,\s*)+)"), - field_value, - RE_FLAGS, + stripped + for stripped in ( + f.strip() + for f in re.findall( + r'((?:[^",]|%s)+)(?=%s|\s*$)' % (rfc9110.quoted_string, r"(?:\s*(?:,\s*)+)"), + field_value, + RE_FLAGS, + ) ) - if f - ] or [] + if stripped + ] def parse_params( @@ -244,6 +297,32 @@ def check_sf_item_token( add_note(invalid_note, value=field_value, **kwargs) +class MEDIA_TYPE_BAD_NAME(Note): + category = categories.GENERAL + level = levels.WARN + _summary = "The %(field_name)s field's media type name isn't registrable." + _text = """\ +[RFC6838](https://www.rfc-editor.org/rfc/rfc6838.html#section-4.2) requires the type and +subtype names in a media type to start with a letter or a digit, and to use only letters, +digits and the characters `!`, `#`, `$`, `&`, `-`, `^`, `_`, `.` and `+`. `%(value)s` +uses something else. + +HTTP's own syntax for media types is more permissive, so recipients are likely to accept +this; however, a media type named this way can't be registered with IANA.""" + + +class MEDIA_TYPE_LONG_NAME(Note): + category = categories.GENERAL + level = levels.WARN + _summary = "The %(field_name)s field's media type name is too long." + _text = """\ +[RFC6838](https://www.rfc-editor.org/rfc/rfc6838.html#section-4.2) limits the type and +subtype names in a media type to 127 characters each; `%(value)s` is longer than that. + +It also recommends keeping them to 64 characters, because longer names run into +implementation limits.""" + + class PARAM_REPEATS(Note): category = categories.GENERAL level = levels.WARN diff --git a/httplint/syntax/__init__.py b/httplint/syntax/__init__.py index d64a1fe..f2f04a4 100755 --- a/httplint/syntax/__init__.py +++ b/httplint/syntax/__init__.py @@ -10,6 +10,7 @@ "rfc5322", "rfc5646", "rfc5987", + "rfc6838", "rfc8288", "rfc9110", "rfc9111", diff --git a/httplint/syntax/rfc6838.py b/httplint/syntax/rfc6838.py new file mode 100644 index 0000000..1c6b3a6 --- /dev/null +++ b/httplint/syntax/rfc6838.py @@ -0,0 +1,42 @@ +""" +Regex for RFC6838 +""" + +# pylint: disable=invalid-name + +from .rfc5234 import ( + ALPHA, + DIGIT, +) + +SPEC_URL = "https://www.rfc-editor.org/rfc/rfc6838" + + +# restricted-name-first = ALPHA / DIGIT + +restricted_name_first = rf"(?: {ALPHA} | {DIGIT} )" + +# restricted-name-chars = ALPHA / DIGIT / "!" / "#" / +# "$" / "&" / "-" / "^" / "_" +# restricted-name-chars =/ "." ; Characters before first dot always +# ; specify a facet name +# restricted-name-chars =/ "+" ; Characters after last plus always +# ; specify a structured syntax suffix + +restricted_name_chars = rf"(?: {ALPHA} | {DIGIT} | ! | \# | \$ | & | \- | \^ | _ | \. | \+ )" + +# restricted-name = restricted-name-first *126restricted-name-chars + +restricted_name = rf"(?: {restricted_name_first} {restricted_name_chars}{{0,126}} )" + +# type-name = restricted-name + +type_name = restricted_name + +# subtype-name = restricted-name + +subtype_name = restricted_name + +# The longest a type-name or subtype-name can be, per restricted-name above. + +RESTRICTED_NAME_MAX_LEN = 127 From 0e1abbb10bd68ec8824c8e03c922683090fc1750 Mon Sep 17 00:00:00 2001 From: Mark Nottingham Date: Sat, 8 Aug 2026 15:31:00 +1000 Subject: [PATCH 2/4] Changed: parse Accept-Query as a Structured Field, per RFC 10008 Accept-Query was implemented as a comma-separated list of media types, but it isn't one. RFC 10008, Section 3 defines it as a List Structured Field whose members are Tokens or Strings, each naming a media range without parameters; media type parameters are carried as Structured Field parameters. Its own example, which the old code mis-parsed, is: Accept-Query: "application/jsonpath", application/sql;charset="UTF-8" Rebase the field on StructuredField with sf_type = "list", and check each member with check_media_type(). Members that are neither Tokens nor Strings are reported with the new ACCEPT_QUERY_BAD_TYPE. Wildcards are allowed -- the RFC permits "*/*" and "xxxx/*" -- so ACCEPT_QUERY_BAD_SYNTAX now talks about media ranges rather than media types. This changes the field's parsed value from a list of (media type, params) tuples to the Structured Field list that http_sf returns. The QUERY method was published as RFC 10008 in June 2026, so the reference moves from the datatracker copy of draft-ietf-httpbis-safe-method-w-body to the RFC. Co-Authored-By: Claude Opus 5 --- httplint/field/parsers/accept_query.py | 75 +++++++++++++++++++++----- 1 file changed, 61 insertions(+), 14 deletions(-) diff --git a/httplint/field/parsers/accept_query.py b/httplint/field/parsers/accept_query.py index 9fc8d1e..111a0bf 100644 --- a/httplint/field/parsers/accept_query.py +++ b/httplint/field/parsers/accept_query.py @@ -1,20 +1,20 @@ -from typing import Tuple +from http_sf import Token -from httplint.field.list_field import HttpListField +from httplint.field.structured_field import StructuredField from httplint.field.tests import FieldTest -from httplint.field.utils import parse_media_type +from httplint.field.utils import check_media_type from httplint.note import Note, categories, levels from httplint.types import ( AddNoteMethodType, NoteClassListType, - ParamDictType, ResponseLinterProtocol, + SFListType, ) -SPEC_URL = "https://datatracker.ietf.org/doc/html/draft-ietf-httpbis-safe-method-w-body" +SPEC_URL = "https://www.rfc-editor.org/rfc/rfc10008.html" -class accept_query(HttpListField[ResponseLinterProtocol]): +class accept_query(StructuredField[ResponseLinterProtocol]): canonical_name = "Accept-Query" description = """\ The `Accept-Query` response header advertises which media types are accepted by the server in the @@ -23,19 +23,45 @@ class accept_query(HttpListField[ResponseLinterProtocol]): syntax = False # Structured Field category = categories.GENERAL deprecated = False + sf_type = "list" + value: SFListType - def parse(self, field_value: str, add_note: AddNoteMethodType) -> Tuple[str, ParamDictType]: - return parse_media_type(field_value, add_note, ACCEPT_QUERY_BAD_SYNTAX, self.reference) + def evaluate(self, add_note: AddNoteMethodType) -> None: + for item in self.value: + # SF List items are (value, parameters) tuples + val = item[0] + if not isinstance(val, (Token, str)): + add_note(ACCEPT_QUERY_BAD_TYPE, value=str(val)) + continue + # Media type parameters are carried as SF parameters, so the item + # value is the media range on its own. + check_media_type( + str(val).lower(), + add_note, + ACCEPT_QUERY_BAD_SYNTAX, + self.reference, + allow_wildcard=True, + ) + + +class ACCEPT_QUERY_BAD_TYPE(Note): + category = categories.GENERAL + level = levels.BAD + _summary = "The Accept-Query header contains a value that isn't a media range." + _text = """\ +`Accept-Query` is a List Structured Field whose members are Tokens or Strings, each +naming a media range accepted in the content of a QUERY request. `%(value)s` is +neither, so it will be ignored.""" class ACCEPT_QUERY_BAD_SYNTAX(Note): category = categories.GENERAL level = levels.BAD - _summary = "The Accept-Query header contains a value that is not a media type." + _summary = "The Accept-Query header contains a value that is not a media range." _text = """\ -`%(value)s` is not a valid media type. `Accept-Query` is a list of media types -(e.g., `application/sparql-query`) accepted in the content of a QUERY request; -see [its definition](%(ref_uri)s) for more information.""" +`%(value)s` is not a valid media range. `Accept-Query` is a list of media ranges +(e.g., `application/sparql-query`, `text/*`) accepted in the content of a QUERY +request; see [its definition](%(ref_uri)s) for more information.""" class AcceptQueryTest(FieldTest[ResponseLinterProtocol]): @@ -44,10 +70,24 @@ class AcceptQueryTest(FieldTest[ResponseLinterProtocol]): expected_out = [("application/sparql-query", {}), ("application/sql", {})] +class AcceptQueryStringTest(FieldTest[ResponseLinterProtocol]): + "Media types that aren't valid Tokens have to be sent as Strings." + + name = "Accept-Query" + inputs = [b'"application/jsonpath", "3d/example"'] + expected_out = [("application/jsonpath", {}), ("3d/example", {})] + + class AcceptQueryParamsTest(FieldTest[ResponseLinterProtocol]): name = "Accept-Query" - inputs = [b"application/example;version=1"] - expected_out = [("application/example", {"version": "1"})] + inputs = [b'application/sql;charset="UTF-8"'] + expected_out = [("application/sql", {"charset": "UTF-8"})] + + +class AcceptQueryWildcardTest(FieldTest[ResponseLinterProtocol]): + name = "Accept-Query" + inputs = [b"*/*, text/*"] + expected_out = [("*/*", {}), ("text/*", {})] class AcceptQueryBadTest(FieldTest[ResponseLinterProtocol]): @@ -55,3 +95,10 @@ class AcceptQueryBadTest(FieldTest[ResponseLinterProtocol]): inputs = [b"invalid"] expected_out = [("invalid", {})] expected_notes: NoteClassListType = [ACCEPT_QUERY_BAD_SYNTAX] + + +class AcceptQueryBadTypeTest(FieldTest[ResponseLinterProtocol]): + name = "Accept-Query" + inputs = [b"123"] + expected_out = [(123, {})] + expected_notes: NoteClassListType = [ACCEPT_QUERY_BAD_TYPE] From 42c2c86724b5454feb5a7e2e70ac746ab240b2cc Mon Sep 17 00:00:00 2001 From: Mark Nottingham Date: Sat, 8 Aug 2026 15:31:08 +1000 Subject: [PATCH 3/4] Fixed: substitute the media type into the Accept q value note BAD_Q_VALUE's summary used str.format syntax, but Note summaries are %-formatted, so it rendered literally as: The q value on '{media_type}' is invalid. Co-Authored-By: Claude Opus 5 --- httplint/field/parsers/accept.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/httplint/field/parsers/accept.py b/httplint/field/parsers/accept.py index 6bebe09..fd2c048 100644 --- a/httplint/field/parsers/accept.py +++ b/httplint/field/parsers/accept.py @@ -66,7 +66,7 @@ def evaluate(self, add_note: AddNoteMethodType) -> None: class BAD_Q_VALUE(Note): category = categories.CONNEG level = levels.WARN - _summary = "The q value on '{media_type}' is invalid." + _summary = "The q value on '%(media_type)s' is invalid." _text = """\ The `q` parameter must be a decimal number between 0 and 1, with at most 3 digits of precision.""" From 927fbc50eed2eb8b76cd30c8e8a4f7f45a507550 Mon Sep 17 00:00:00 2001 From: Mark Nottingham Date: Sat, 8 Aug 2026 15:51:51 +1000 Subject: [PATCH 4/4] report non-token media type names where there's no syntax check check_media_type() dropped names that aren't valid HTTP tokens, on the grounds that the field's own ABNF check reports them. That doesn't hold for Accept-Query, which has syntax = False because it's a Structured Field, so nothing reported them at all: Accept-Query: "text/pl(in" -> no notes Accept-Query: "text/pl in" -> no notes RFC 10008 Strings exist precisely to carry names that aren't valid Tokens, so that's where a bad name is most likely to turn up. Add a check_token argument for callers with no ABNF backstop, which reports such names with the field's own bad-syntax note, and set it in accept_query. Also drop the bare "*" escape hatch. Neither RFC 9110's media-range, LDP's nor RFC 10008's permits it; for Accept and Accept-Post the field's ABNF masked it, but "Accept-Query: *" passed silently. While here: * Rename RESTRICTED_NAME_CHARS to RESTRICTED_NAME_UNBOUNDED. It's a whole restricted-name, not a character class, and sat one line from rfc6838.restricted_name_chars, which is one. * Lowercase the media ranges accept_query stores, as the other media-type fields do; RFC 10008 members are case-insensitive. * Cover a String member with a non-token name, a bare "*", case normalisation, and a bad name in the type half rather than the subtype. Co-Authored-By: Claude Opus 5 --- httplint/field/parsers/accept_query.py | 38 ++++++++++++++++++++++++-- httplint/field/parsers/content_type.py | 9 ++++++ httplint/field/utils.py | 30 +++++++++++++------- 3 files changed, 64 insertions(+), 13 deletions(-) diff --git a/httplint/field/parsers/accept_query.py b/httplint/field/parsers/accept_query.py index 111a0bf..16c836a 100644 --- a/httplint/field/parsers/accept_query.py +++ b/httplint/field/parsers/accept_query.py @@ -27,21 +27,29 @@ class accept_query(StructuredField[ResponseLinterProtocol]): value: SFListType def evaluate(self, add_note: AddNoteMethodType) -> None: + normalised: SFListType = [] for item in self.value: # SF List items are (value, parameters) tuples - val = item[0] + val, params = item if not isinstance(val, (Token, str)): add_note(ACCEPT_QUERY_BAD_TYPE, value=str(val)) + normalised.append(item) continue # Media type parameters are carried as SF parameters, so the item - # value is the media range on its own. + # value is the media range on its own. Media ranges are + # case-insensitive, so store them lowercased, as the other + # media-type fields do. + media_range = str(val).lower() check_media_type( - str(val).lower(), + media_range, add_note, ACCEPT_QUERY_BAD_SYNTAX, self.reference, allow_wildcard=True, + check_token=True, # no syntax check on a Structured Field ) + normalised.append((type(val)(media_range), params)) + self.value = normalised class ACCEPT_QUERY_BAD_TYPE(Note): @@ -90,6 +98,12 @@ class AcceptQueryWildcardTest(FieldTest[ResponseLinterProtocol]): expected_out = [("*/*", {}), ("text/*", {})] +class AcceptQueryCaseTest(FieldTest[ResponseLinterProtocol]): + name = "Accept-Query" + inputs = [b'APPLICATION/SQL, "TEXT/Plain"'] + expected_out = [("application/sql", {}), ("text/plain", {})] + + class AcceptQueryBadTest(FieldTest[ResponseLinterProtocol]): name = "Accept-Query" inputs = [b"invalid"] @@ -97,6 +111,24 @@ class AcceptQueryBadTest(FieldTest[ResponseLinterProtocol]): expected_notes: NoteClassListType = [ACCEPT_QUERY_BAD_SYNTAX] +class AcceptQueryBadStringTest(FieldTest[ResponseLinterProtocol]): + "A String member can carry a name that isn't a valid HTTP token." + + name = "Accept-Query" + inputs = [b'"text/pl in"'] + expected_out = [("text/pl in", {})] + expected_notes: NoteClassListType = [ACCEPT_QUERY_BAD_SYNTAX] + + +class AcceptQueryBareStarTest(FieldTest[ResponseLinterProtocol]): + "Only */* and type/* are permitted, not a bare *." + + name = "Accept-Query" + inputs = [b"*"] + expected_out = [("*", {})] + expected_notes: NoteClassListType = [ACCEPT_QUERY_BAD_SYNTAX] + + class AcceptQueryBadTypeTest(FieldTest[ResponseLinterProtocol]): name = "Accept-Query" inputs = [b"123"] diff --git a/httplint/field/parsers/content_type.py b/httplint/field/parsers/content_type.py index f116e8a..bdf572f 100644 --- a/httplint/field/parsers/content_type.py +++ b/httplint/field/parsers/content_type.py @@ -51,6 +51,15 @@ class CTBadNameTest(FieldTest[AnyMessageLinterProtocol]): expected_notes: NoteClassListType = [MEDIA_TYPE_BAD_NAME] +class CTBadTypeNameTest(FieldTest[AnyMessageLinterProtocol]): + "The type half is checked as well as the subtype half." + + name = "Content-Type" + inputs = [b"~text/plain"] + expected_out: Any = ("~text/plain", {}) + expected_notes: NoteClassListType = [MEDIA_TYPE_BAD_NAME] + + class CTBadNameFirstTest(FieldTest[AnyMessageLinterProtocol]): "RFC 6838 names have to start with a letter or a digit." diff --git a/httplint/field/utils.py b/httplint/field/utils.py index d2e2e8c..2098d58 100644 --- a/httplint/field/utils.py +++ b/httplint/field/utils.py @@ -16,7 +16,9 @@ # over-long names are reported as MEDIA_TYPE_LONG_NAME instead of being lumped in # with character errors. -RESTRICTED_NAME_CHARS = rf"(?: {rfc6838.restricted_name_first} {rfc6838.restricted_name_chars}* )" +RESTRICTED_NAME_UNBOUNDED = ( + rf"(?: {rfc6838.restricted_name_first} {rfc6838.restricted_name_chars}* )" +) def parse_media_type( @@ -50,6 +52,7 @@ def check_media_type( bad_syntax_note: Optional[Type[Note]] = None, ref_uri: Optional[str] = None, allow_wildcard: bool = False, + check_token: bool = False, ) -> None: """ Check a media-type against the shape HTTP gives it and the naming rules in @@ -57,8 +60,13 @@ def check_media_type( If ``bad_syntax_note`` is provided, it is emitted (with ``ref_uri`` if given) when the value isn't shaped like a media-type at all. When ``allow_wildcard`` - is true the value is treated as a media range, so ``*``, ``*/*`` and - ``type/*`` are accepted; otherwise a ``*`` in either position is an error. + is true the value is treated as a media range, so ``*/*`` and ``type/*`` are + accepted; otherwise a ``*`` in either position is an error. + + Names that aren't valid HTTP tokens are normally left to the field's own + syntax check. Callers without one -- Structured Fields, whose members can + carry any string -- should set ``check_token`` so that they're reported here + instead. """ def bad_syntax() -> None: @@ -69,9 +77,6 @@ def bad_syntax() -> None: kwargs["ref_uri"] = ref_uri add_note(bad_syntax_note, **kwargs) - if allow_wildcard and media_type == "*": - return - type_name, slash, subtype_name = media_type.partition("/") if not (slash and type_name and subtype_name) or "/" in subtype_name: bad_syntax() @@ -85,13 +90,18 @@ def bad_syntax() -> None: return names = [name for name in names if name != "*"] - # Names that aren't valid HTTP tokens are already reported by the field's own - # syntax check; RFC 6838 only adds information for the ones that are. - names = [name for name in names if re.match(rf"^{rfc9110.token}$", name, RE_FLAGS)] + tokens = [name for name in names if re.match(rf"^{rfc9110.token}$", name, RE_FLAGS)] + if len(tokens) != len(names): + if check_token: + bad_syntax() + return + # Otherwise the field's own syntax check reports it, and RFC 6838 has + # nothing to add. + names = tokens if any(len(name) > rfc6838.RESTRICTED_NAME_MAX_LEN for name in names): add_note(MEDIA_TYPE_LONG_NAME, value=media_type) - if any(not re.match(rf"^{RESTRICTED_NAME_CHARS}$", name, RE_FLAGS) for name in names): + if any(not re.match(rf"^{RESTRICTED_NAME_UNBOUNDED}$", name, RE_FLAGS) for name in names): add_note(MEDIA_TYPE_BAD_NAME, value=media_type)