diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..2f89862 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,23 @@ +name: TypeChecks +on: push + +jobs: + lint: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.12", "3.13", "3.14"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + cache-dependency-path: requirements-dev.txt + allow-prereleases: true + - run: pip install -r requirements-dev.txt + - run: pyflakes mureq.py + - run: flake8 mureq.py + - run: mypy mureq.py + - run: pyrefly check mureq.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..1f4e15a --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,17 @@ +name: Test +on: push + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.12", "3.13", "3.14"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + - run: python -m unittest tests.test_unit diff --git a/CHANGELOG.md b/CHANGELOG.md index d93db84..12e8b15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,20 @@ # Changelog All notable changes to mureq will be documented in this file. +## [0.3.0] - 2026-03-16 + +v0.3.0 is the third release of mureq. + +### API breaks +* Repeated headers in `Response.headers` are now joined with `, ` instead of `,`, matching the Requests behavior + +### Fixed +* Redirect handling of [303 See Other](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/303) now clears the request body + +### Added +* Added `Response.raw_headers`, which contains the original unjoined headers as a list of string pairs +* Added type annotations (thanks [@hbmartin](https://github.com/hbmartin)!) + ## [0.2.0] - 2022-02-03 v0.2.0 is the second release of mureq. diff --git a/README.md b/README.md index 60bfceb..14d6b7e 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ However, the API design of python-requests is excellent and in my opinion still mureq supports Python 3.6 and higher. Copy `mureq.py` into a suitable directory of your project, then import as you would any other internal module, e.g. `import .mureq` or `import bar.baz.mureq`. -Supply-chain attacks are considerably mitigated simply by vendoring mureq (i.e. copying it into your tree). If you are also concerned about future attacks on this GitHub account (or GitHub itself), tagged releases of mureq will be signed with the GPG key `0x740FC947B135E7627D4D00F21996B89DF018DCAB` (expires 2025-07-28), or some future key in a chain of trust from it. +Supply-chain attacks are considerably mitigated simply by vendoring mureq (i.e. copying it into your tree). If you are also concerned about future attacks on this GitHub account (or GitHub itself), tagged releases of mureq will be signed with the GPG key `0x740FC947B135E7627D4D00F21996B89DF018DCAB` (expires 2030-07-24), or some future key in a chain of trust from it. Vendoring mureq's tests is not recommended. The tests rely on third-party HTTP services, so including them in a project-specific test suite or CI/CD pipeline will reduce the reliability of your project's tests and also risks overburdening the third-party services. @@ -82,11 +82,11 @@ The core API (`mureq.get`, `mureq.post`, `mureq.request`, etc.) is similar to py If you're switching from python-requests, there are a few things to keep in mind: 1. `mureq.get`, `mureq.post`, and `mureq.request` mostly work like the [analogous python-requests calls](https://docs.python-requests.org/en/latest/user/quickstart/#make-a-request). -1. The response type is `mureq.HTTPResponse`, which exposes fewer methods and properties than `requests.Response`. In particular, it does not have `text` (since mureq doesn't do any encoding detection). Instead, the response body is in the `body` member, which is always of type `bytes`. (For the sake of compatibility, the `content` property is provided as an alias for `body`.) +1. The response type is `mureq.Response`, which exposes fewer methods and properties than `requests.Response`. In particular, it does not have `text` (since mureq doesn't do any encoding detection). Instead, the response body is in the `body` member, which is always of type `bytes`. (For the sake of compatibility, the `content` property is provided as an alias for `body`.) 1. The default way to send a POST body is with the `body` kwarg, which only accepts `bytes`. 1. The `json` kwarg takes an arbitrary object, which is serialized to JSON, encoded as UTF-8, and sent as the request body with the usual `Content-Type: application/json` header. 1. To send a form-encoded POST body, use the `form` kwarg. This accepts a dictionary of key-value pairs, or any object that can be serialized by [urllib.parse.urlencode](https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode). It will add the usual `Content-Type: application/x-www-form-urlencoded` header. -1. To make a request without reading the entire body at once, use `with mureq.yield_response(url, method, **kwargs)`. This yields a [http.client.HTTPResponse](https://docs.python.org/3/library/http.client.html#httpresponse-objects). Exiting the contextmanager automatically closes the socket. +1. To make a request without reading the entire body at once, use `with mureq.yield_response(method, url, **kwargs)`. This yields a [http.client.HTTPResponse](https://docs.python.org/3/library/http.client.html#httpresponse-objects). Exiting the contextmanager automatically closes the socket. 1. mureq does not follow HTTP redirections by default. To enable them, use the kwarg `max_redirects`, which takes an integer number of redirects to allow, e.g. `max_redirects=2`. 1. mureq will throw a subclass of `mureq.HTTPException` (which is actually just [http.client.HTTPException](https://docs.python.org/3/library/http.client.html#http.client.HTTPException)) for any runtime I/O error (including invalid HTTP responses, connection failures, timeouts, and exceeding the redirection limit). It may throw other exceptions (in particular `ValueError`) for programming errors, such as invalid or inconsistent arguments. 1. mureq supports two ways of making HTTP requests over a Unix domain stream socket: diff --git a/mureq.py b/mureq.py index 49547e4..9777cf7 100644 --- a/mureq.py +++ b/mureq.py @@ -5,6 +5,7 @@ mureq is copyright 2021 by its contributors and is released under the 0BSD ("zero-clause BSD") license. """ +# fmt: off import contextlib import io import os.path @@ -12,20 +13,24 @@ import ssl import sys import urllib.parse -from http.client import HTTPConnection, HTTPSConnection, HTTPMessage, HTTPException +from collections.abc import Generator, MutableMapping +from http.client import HTTPConnection, HTTPSConnection, HTTPMessage, HTTPException, HTTPResponse +from typing import Any, cast -__version__ = '0.2.0' +__version__ = '0.3.0' __all__ = ['HTTPException', 'TooManyRedirects', 'Response', 'yield_response', 'request', 'get', 'post', 'head', 'put', 'patch', 'delete'] -DEFAULT_TIMEOUT = 15.0 +DEFAULT_TIMEOUT: float = 15.0 # e.g. "Python 3.8.10" -DEFAULT_UA = "Python " + sys.version.split()[0] +DEFAULT_UA: str = "Python " + sys.version.split()[0] +Headers = MutableMapping[str, str] | HTTPMessage -def request(method, url, *, read_limit=None, **kwargs): + +def request(method: str, url: str, *, read_limit: int | None = None, **kwargs) -> "Response": """request performs an HTTP request and reads the entire response body. :param str method: HTTP method to request (e.g. 'GET', 'POST') @@ -42,45 +47,59 @@ def request(method, url, *, read_limit=None, **kwargs): body = response.read(read_limit) except HTTPException: raise - except IOError as e: + except OSError as e: raise HTTPException(str(e)) from e - return Response(response.url, response.status, _prepare_incoming_headers(response.headers), body) + headers, raw_headers = _prepare_incoming_headers(response.headers) + return Response(response.url, response.status, headers, raw_headers, body) -def get(url, **kwargs): +def get(url: str, **kwargs) -> "Response": """get performs an HTTP GET request.""" return request('GET', url=url, **kwargs) -def post(url, body=None, **kwargs): +def post(url: str, body: bytes | None = None, **kwargs) -> "Response": """post performs an HTTP POST request.""" return request('POST', url=url, body=body, **kwargs) -def head(url, **kwargs): +def head(url: str, **kwargs) -> "Response": """head performs an HTTP HEAD request.""" return request('HEAD', url=url, **kwargs) -def put(url, body=None, **kwargs): +def put(url: str, body: bytes | None = None, **kwargs) -> "Response": """put performs an HTTP PUT request.""" return request('PUT', url=url, body=body, **kwargs) -def patch(url, body=None, **kwargs): +def patch(url: str, body: bytes | None = None, **kwargs) -> "Response": """patch performs an HTTP PATCH request.""" return request('PATCH', url=url, body=body, **kwargs) -def delete(url, **kwargs): +def delete(url: str, **kwargs) -> "Response": """delete performs an HTTP DELETE request.""" return request('DELETE', url=url, **kwargs) @contextlib.contextmanager -def yield_response(method, url, *, unix_socket=None, timeout=DEFAULT_TIMEOUT, headers=None, - params=None, body=None, form=None, json=None, verify=True, source_address=None, - max_redirects=None, ssl_context=None): +def yield_response( + method: str, + url: str, + *, + unix_socket: str | None = None, + timeout: float | None = DEFAULT_TIMEOUT, + headers: Headers | list[tuple[str, str]] | None = None, + params: dict[str, str | bytes] | list[tuple[str, str | bytes]] | None = None, + body: bytes | None = None, + form: dict[str, str | bytes] | list[tuple[str, str | bytes]] | None = None, + json: Any = None, + verify: bool = True, + source_address: str | tuple[str, int] | None = None, + max_redirects: int | None = None, + ssl_context: ssl.SSLContext | None = None, +) -> Generator[HTTPResponse, None, None]: """yield_response is a low-level API that exposes the actual http.client.HTTPResponse via a contextmanager. @@ -118,7 +137,7 @@ def yield_response(method, url, *, unix_socket=None, timeout=DEFAULT_TIMEOUT, he enc_params = _prepare_params(params) body = _prepare_body(body, form, json, headers) - visited_urls = [] + visited_urls: list[str] = [] while max_redirects is None or len(visited_urls) <= max_redirects: url, conn, path = _prepare_request(method, url, enc_params=enc_params, timeout=timeout, unix_socket=unix_socket, verify=verify, source_address=source_address, ssl_context=ssl_context) @@ -126,12 +145,12 @@ def yield_response(method, url, *, unix_socket=None, timeout=DEFAULT_TIMEOUT, he visited_urls.append(url) try: try: - conn.request(method, path, headers=headers, body=body) + conn.request(method, path, headers=cast(Any, headers), body=body) response = conn.getresponse() except HTTPException: raise - except IOError as e: - # wrap any IOError that is not already an HTTPException + except OSError as e: + # wrap any OSError that is not already an HTTPException # in HTTPException, exposing a uniform API for remote errors raise HTTPException(str(e)) from e redirect_url = _check_redirect(url, response.status, response.headers) @@ -144,6 +163,7 @@ def yield_response(method, url, *, unix_socket=None, timeout=DEFAULT_TIMEOUT, he if response.status == 303: # 303 See Other: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/303 method = 'GET' + body = None finally: conn.close() @@ -156,41 +176,48 @@ class Response: :ivar str url: the retrieved URL, indicating whether a redirection occurred :ivar int status_code: the HTTP status code :ivar http.client.HTTPMessage headers: the HTTP headers + :ivar raw_headers: the original unmerged HTTP headers as a list of tuples :ivar bytes body: the payload body of the response """ - __slots__ = ('url', 'status_code', 'headers', 'body') + __slots__ = ('url', 'status_code', 'headers', 'raw_headers', 'body') + url: str + status_code: int + headers: Headers + raw_headers: list[tuple[str, str]] + body: bytes - def __init__(self, url, status_code, headers, body): - self.url, self.status_code, self.headers, self.body = url, status_code, headers, body + def __init__(self, url: str, status_code: int, headers: Headers, raw_headers: list[tuple[str, str]], body: bytes): + self.url, self.status_code, self.headers, self.raw_headers, self.body = \ + url, status_code, headers, raw_headers, body - def __repr__(self): + def __repr__(self) -> str: return f"Response(status_code={self.status_code:d})" @property - def ok(self): + def ok(self) -> bool: """ok returns whether the response had a successful status code (anything other than a 40x or 50x).""" return not (400 <= self.status_code < 600) @property - def content(self): + def content(self) -> bytes: """content returns the response body (the `body` member). This is an alias for compatibility with requests.Response.""" return self.body - def raise_for_status(self): + def raise_for_status(self) -> None: """raise_for_status checks the response's success code, raising an exception for error codes.""" if not self.ok: raise HTTPErrorStatus(self.status_code) - def json(self): + def json(self) -> Any: """Attempts to deserialize the response body as UTF-8 encoded JSON.""" import json as jsonlib return jsonlib.loads(self.body) - def _debugstr(self): + def _debugstr(self) -> str: buf = io.StringIO() print("HTTP", self.status_code, file=buf) for k, v in self.headers.items(): @@ -216,10 +243,10 @@ class HTTPErrorStatus(HTTPException): called explicitly. """ - def __init__(self, status_code): + def __init__(self, status_code: int): self.status_code = status_code - def __str__(self): + def __str__(self) -> str: return f"HTTP response returned error code {self.status_code:d}" @@ -249,7 +276,7 @@ def connect(self): self.sock = sock -def _check_redirect(url, status, response_headers): +def _check_redirect(url: str, status: int, response_headers: HTTPMessage) -> str | None: """Return the URL to redirect to, or None for no redirection.""" if status not in (301, 302, 303, 307, 308): return None @@ -276,7 +303,7 @@ def _check_redirect(url, status, response_headers): parsed_location.query, parsed_location.fragment)) -def _prepare_outgoing_headers(headers): +def _prepare_outgoing_headers(headers: Headers | list[tuple[str, str]] | None) -> HTTPMessage: if headers is None: headers = HTTPMessage() elif not isinstance(headers, HTTPMessage): @@ -294,16 +321,18 @@ def _prepare_outgoing_headers(headers): # XXX join multi-headers together so that get(), __getitem__(), # etc. behave intuitively, then stuff them back in an HTTPMessage. -def _prepare_incoming_headers(headers): - headers_dict = {} +def _prepare_incoming_headers(headers: HTTPMessage) -> tuple[HTTPMessage, list[tuple[str, str]]]: + raw_headers: list[tuple[str, str]] = [] + headers_dict: dict[str, list[str]] = {} for k, v in headers.items(): headers_dict.setdefault(k, []).append(v) + raw_headers.append((k, v)) result = HTTPMessage() # note that iterating over headers_dict preserves the original # insertion order in all versions since Python 3.6: for k, vlist in headers_dict.items(): - result[k] = ','.join(vlist) - return result + result[k] = ', '.join(vlist) + return result, raw_headers def _setdefault_header(headers, name, value): @@ -311,7 +340,7 @@ def _setdefault_header(headers, name, value): headers[name] = value -def _prepare_body(body, form, json, headers): +def _prepare_body(body, form, json, headers) -> bytes | None: if body is not None: if not isinstance(body, bytes): raise TypeError('body must be bytes or None', type(body)) @@ -324,7 +353,7 @@ def _prepare_body(body, form, json, headers): if form is not None: _setdefault_header(headers, 'Content-Type', _FORM_CONTENTTYPE) - return urllib.parse.urlencode(form, doseq=True) + return urllib.parse.urlencode(form, doseq=True).encode('ascii') return None @@ -352,6 +381,8 @@ def _prepare_request(method, url, *, enc_params='', timeout=DEFAULT_TIMEOUT, sou is_https = (scheme == 'https') host = parsed_url.hostname + if host is None: + raise ValueError("host is missing from url", url) port = 443 if is_https else 80 if parsed_url.port: port = parsed_url.port diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..430ecbf --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,4 @@ +pyrefly==0.56.0 +mypy==1.19.1 +pyflakes==3.3.2 +flake8==7.2.0 diff --git a/tests/test_integration.py b/tests/test_integration.py index fbd391e..1da4bb6 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -12,14 +12,25 @@ import threading import tempfile import os.path -import urllib.parse import ssl import http.client import http.server +import urllib.request +import urllib.parse import mureq +def strip_b64_if_present(data): + # if httpbingo isn't sure about the incoming content-type header, it returns the data field + # as a b64-encoded data url: data:application/jose+json;base64,YT0x + # if it's sure the content type is acceptable in a JSON string, it returns the data itself: a=1 + if data.startswith('data:'): + with urllib.request.urlopen(data) as response: + return response.read().decode('utf8') + return data + + class MureqIntegrationTestCase(unittest.TestCase): def _get_json(self, response): @@ -114,25 +125,33 @@ def test_url_populated(self): def test_head(self): response = mureq.head('https://httpbingo.org/head') - self.assertIn('Content-Length', response.headers) + date_header = response.headers['Date'] + self.assertTrue(date_header) + # check raw_headers as well + success = False + for k, v in response.raw_headers: + if k.lower() == 'date': + success = True + self.assertEqual(v, date_header) + self.assertTrue(success, 'headers and raw_headers do not correspond') def test_post(self): result = self._get_json(mureq.post('https://httpbingo.org/post', body=b'xyz')) self.assertEqual(result['headers']['User-Agent'], [mureq.DEFAULT_UA]) self.assertEqual(result['url'], 'https://httpbingo.org/post') - self.assertEqual(result['data'], 'xyz') + self.assertEqual(strip_b64_if_present(result['data']), 'xyz') def test_put(self): result = self._get_json(mureq.put('https://httpbingo.org/put', body=b'strawberry')) self.assertEqual(result['headers']['User-Agent'], [mureq.DEFAULT_UA]) self.assertEqual(result['url'], 'https://httpbingo.org/put') - self.assertEqual(result['data'], 'strawberry') + self.assertEqual(strip_b64_if_present(result['data']), 'strawberry') def test_patch(self): result = self._get_json(mureq.patch('https://httpbingo.org/patch', body=b'burrito')) self.assertEqual(result['headers']['User-Agent'], [mureq.DEFAULT_UA]) self.assertEqual(result['url'], 'https://httpbingo.org/patch') - self.assertEqual(result['data'], 'burrito') + self.assertEqual(strip_b64_if_present(result['data']), 'burrito') def test_json(self): result = self._get_json(mureq.post('https://httpbingo.org/post', json={'a': 1})) @@ -145,7 +164,7 @@ def test_json(self): headers={'Content-Type': 'application/jose+json'})) # we must not override the user-supplied content-type header self.assertEqual(result['headers']['Content-Type'], ['application/jose+json']) - self.assertEqual(json.loads(result['data']), obj) + self.assertEqual(json.loads(strip_b64_if_present(result['data'])), obj) def test_form(self): result = self._get_json(mureq.post('https://httpbingo.org/post', form={'a': '1'})) @@ -158,7 +177,7 @@ def test_form(self): result = self._get_json(mureq.post('https://httpbingo.org/post', form={'a': '1'}, headers={'Content-Type': 'application/jose+json'})) self.assertEqual(result['headers']['Content-Type'], ['application/jose+json']) - self.assertEqual(result['data'], 'a=1') + self.assertEqual(strip_b64_if_present(result['data']), 'a=1') def test_redirects(self): # redirects us to /get @@ -207,7 +226,7 @@ def test_307(self): response = mureq.post('https://httpbingo.org/redirect-to?url=/post&status_code=307', body=b'xyz', max_redirects=1) self.assertEqual(response.url, 'https://httpbingo.org/post') self.assertEqual(response.status_code, 200) - self.assertEqual(json.loads(response.body)['data'], 'xyz') + self.assertEqual(strip_b64_if_present(response.json()['data']), 'xyz') def test_303(self): # 303 turns POST into GET diff --git a/tests/test_unit.py b/tests/test_unit.py index 34ca668..edda600 100644 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -34,17 +34,17 @@ def test_302_relative_noslash(self): class ReponseTestCase(unittest.TestCase): def test_ok(self): - self.assertEqual(Response('', 200, HTTPMessage(), b'').ok, True) - self.assertEqual(Response('', 204, HTTPMessage(), b'').ok, True) - self.assertEqual(Response('', 301, HTTPMessage(), b'').ok, True) - self.assertEqual(Response('', 400, HTTPMessage(), b'').ok, False) - self.assertEqual(Response('', 404, HTTPMessage(), b'').ok, False) - self.assertEqual(Response('', 418, HTTPMessage(), b'').ok, False) - self.assertEqual(Response('', 500, HTTPMessage(), b'').ok, False) - self.assertEqual(Response('', 504, HTTPMessage(), b'').ok, False) + self.assertEqual(Response('', 200, HTTPMessage(), [], b'').ok, True) + self.assertEqual(Response('', 204, HTTPMessage(), [], b'').ok, True) + self.assertEqual(Response('', 301, HTTPMessage(), [], b'').ok, True) + self.assertEqual(Response('', 400, HTTPMessage(), [], b'').ok, False) + self.assertEqual(Response('', 404, HTTPMessage(), [], b'').ok, False) + self.assertEqual(Response('', 418, HTTPMessage(), [], b'').ok, False) + self.assertEqual(Response('', 500, HTTPMessage(), [], b'').ok, False) + self.assertEqual(Response('', 504, HTTPMessage(), [], b'').ok, False) def _assert_raises_for_status(self, code): - resp = Response('', code, HTTPMessage(), b'') + resp = Response('', code, HTTPMessage(), [], b'') try: resp.raise_for_status() except HTTPErrorStatus as e: @@ -53,7 +53,7 @@ def _assert_raises_for_status(self, code): raise AssertionError("did not raise for status", code) def _assert_does_not_raise_for_status(self, code): - resp = Response('', code, HTTPMessage(), b'') + resp = Response('', code, HTTPMessage(), [], b'') resp.raise_for_status() def test_raise_for_status(self):