5
This module contains the primary objects that power Requests.
8
from __future__ import annotations
12
# Import encoding now, to avoid implicit import later.
13
# Implicit import within threads may cause LookupError when standard library is in a ZIP,
14
# such as in Embedded Python. See https://github.com/psf/requests/issues/3578.
15
import encodings.idna # noqa: F401
16
from collections.abc import Callable, Generator, Iterable, Iterator, Mapping
17
from io import UnsupportedOperation
27
from urllib3.exceptions import (
34
from urllib3.fields import RequestField
35
from urllib3.filepost import encode_multipart_formdata
36
from urllib3.util import parse_url
38
from . import _types as _t
39
from ._internal_utils import to_native_string, unicode_is_ascii
40
from .auth import HTTPBasicAuth
51
from .compat import json as complexjson
57
from .exceptions import (
67
from .exceptions import JSONDecodeError as RequestsJSONDecodeError
68
from .exceptions import SSLError as RequestsSSLError
69
from .hooks import default_hooks
70
from .status_codes import codes
71
from .structures import CaseInsensitiveDict
73
check_header_validity,
80
stream_decode_response_unicode,
80
stream_decode_response_unicode,
86
from http.cookiejar import CookieJar
88
from typing_extensions import Self
90
from .adapters import HTTPAdapter
91
from .cookies import RequestsCookieJar
93
#: The set of HTTP status codes that indicate an automatically
94
#: processable redirect.
95
REDIRECT_STATI: Final[tuple[int, ...]] = ( # type: ignore[assignment]
99
codes.temporary_redirect, # 307
100
codes.permanent_redirect, # 308
103
DEFAULT_REDIRECT_LIMIT: int = 30
104
CONTENT_CHUNK_SIZE: int = 10 * 1024
105
ITER_CHUNK_SIZE: int = 512
108
class RequestEncodingMixin:
112
def path_url(self) -> str:
113
"""Build the path URL to use."""
117
p = urlsplit(cast(str, self.url))
134
def _encode_params(data: str) -> str: ...
138
def _encode_params(data: bytes) -> bytes: ...
143
data: _t.SupportsRead[str | bytes],
144
) -> _t.SupportsRead[str | bytes]: ...
143
data: _t.SupportsRead[str | bytes],
144
) -> _t.SupportsRead[str | bytes]: ...
148
def _encode_params(data: _t.KVDataType) -> str: ...
152
data: _t.EncodableDataType,
153
) -> str | bytes | _t.SupportsRead[str | bytes]:
154
"""Encode parameters in a piece of data.
156
Will successfully encode parameters when passed as a dict or a list of
157
2-tuples. Order is retained if data is a list of 2-tuples but arbitrary
158
if parameters are supplied as a dict.
161
if isinstance(data, (str, bytes)):
163
elif _t.has_read(data):
165
elif hasattr(data, "__iter__"):
166
result: list[tuple[bytes, bytes]] = []
167
for k, vs in to_key_val_list(data):
168
if isinstance(vs, basestring) or not hasattr(vs, "__iter__"):
174
k.encode("utf-8") if isinstance(k, str) else k,
175
v.encode("utf-8") if isinstance(v, str) else v,
178
return urlencode(result, doseq=True)
180
return data # type: ignore[return-value] # unreachable for valid _t.DataType
178
return urlencode(result, doseq=True)
180
return data # type: ignore[return-value] # unreachable for valid _t.DataType
184
files: _t.FilesType, data: _t.RawDataType | None
185
) -> tuple[bytes, str]:
186
"""Build the body for a multipart/form-data request.
188
Will successfully encode files when passed as a dict or a list of
189
tuples. Order is retained if data is a list of tuples but arbitrary
190
if parameters are supplied as a dict.
191
The tuples may be 2-tuples (filename, fileobj), 3-tuples (filename, fileobj, contentype)
192
or 4-tuples (filename, fileobj, contentype, custom_headers).
195
raise ValueError("Files must be provided.")
196
elif isinstance(data, basestring):
197
raise ValueError("Data must not be a string.")
199
new_fields: list[RequestField | tuple[str, bytes]] = []
200
fields = to_key_val_list(data or {})
201
files = to_key_val_list(files or {})
203
for field, val in fields:
204
if isinstance(val, basestring) or not hasattr(val, "__iter__"):
208
# Don't call str() on bytestrings: in Py3 it all goes wrong.
209
if not isinstance(v, bytes):
208
# Don't call str() on bytestrings: in Py3 it all goes wrong.
209
if not isinstance(v, bytes):
214
field.decode("utf-8")
215
if isinstance(field, bytes)
217
v.encode("utf-8") if isinstance(v, str) else v,
222
# support for explicit filename
225
if isinstance(v, (tuple, list)):
233
fn = guess_filename(v) or k
236
if isinstance(fp, (str, bytes, bytearray)):
238
elif _t.has_read(fp):
240
elif fp is None: # defensive check for untyped callers
245
rf = RequestField(name=k, data=fdata, filename=fn, headers=fh)
246
rf.make_multipart(content_type=ft)
247
new_fields.append(rf)
245
rf = RequestField(name=k, data=fdata, filename=fn, headers=fh)
246
rf.make_multipart(content_type=ft)
247
new_fields.append(rf)
249
body, content_type = encode_multipart_formdata(new_fields)
251
return body, content_type
254
class RequestHooksMixin:
255
hooks: dict[str, list[_t.HookType]]
258
self, event: str, hook: Iterable[_t.HookType] | _t.HookType
260
"""Properly register a hook."""
262
if event not in self.hooks:
263
raise ValueError(f'Unsupported event specified, with event name "{event}"')
265
if isinstance(hook, Callable):
266
self.hooks[event].append(hook)
267
elif hasattr(hook, "__iter__"):
268
self.hooks[event].extend(
269
h for h in hook if isinstance(h, Callable)
270
) # defensive runtime filter
272
def deregister_hook(self, event: str, hook: _t.HookType) -> bool:
273
"""Deregister a previously registered hook.
274
Returns True if the hook existed, False if not.
278
self.hooks[event].remove(hook)
284
class Request(RequestHooksMixin):
285
"""A user-created :class:`Request <Request>` object.
287
Used to prepare a :class:`PreparedRequest <PreparedRequest>`, which is sent to the server.
284
class Request(RequestHooksMixin):
285
"""A user-created :class:`Request <Request>` object.
287
Used to prepare a :class:`PreparedRequest <PreparedRequest>`, which is sent to the server.
289
:param method: HTTP method to use.
290
:param url: URL to send.
291
:param headers: dictionary of headers to send.
292
:param files: dictionary of {filename: fileobject} files to multipart upload.
293
:param data: the body to attach to the request. If a dictionary or
294
list of tuples ``[(key, value)]`` is provided, form-encoding will
296
:param json: json for the body to attach to the request (if files or data is not specified).
297
:param params: URL parameters to append to the URL. If a dictionary or
298
list of tuples ``[(key, value)]`` is provided, form-encoding will
300
:param auth: Auth handler or (user, pass) tuple.
301
:param cookies: dictionary or CookieJar of cookies to attach to this request.
302
:param hooks: dictionary of callback hooks, for internal usage.
307
>>> req = requests.Request('GET', 'https://httpbin.org/get')
309
<PreparedRequest [GET]>
312
hooks: dict[str, list[_t.HookType]]
314
url: _t.UriType | None
315
headers: Mapping[str, str | bytes]
319
params: _t.ParamsType
321
cookies: RequestsCookieJar | CookieJar | dict[str, str] | None
319
params: _t.ParamsType
321
cookies: RequestsCookieJar | CookieJar | dict[str, str] | None
325
method: str | None = None,
326
url: _t.UriType | None = None,
327
headers: _t.HeadersType = None,
328
files: _t.FilesType = None,
329
data: _t.DataType = None,
330
params: _t.ParamsType = None,
331
auth: _t.AuthType = None,
332
cookies: RequestsCookieJar | CookieJar | dict[str, str] | None = None,
333
hooks: _t.HooksInputType | None = None,
334
json: _t.JsonType = None,
336
# Default empty dicts for dict params.
337
data = [] if data is None else data
338
files = [] if files is None else files
339
headers = {} if headers is None else headers
340
params = {} if params is None else params
341
hooks = {} if hooks is None else hooks
343
self.hooks = default_hooks()
344
for k, v in list(hooks.items()):
345
self.register_hook(event=k, hook=v)
349
self.headers = headers
355
self.cookies = cookies
357
def __repr__(self) -> str:
358
return f"<Request [{self.method}]>"
360
def prepare(self) -> PreparedRequest:
361
"""Constructs a :class:`PreparedRequest <PreparedRequest>` for transmission and returns it."""
362
p = PreparedRequest()
366
headers=self.headers,
372
cookies=self.cookies,
372
cookies=self.cookies,
378
class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
379
"""The fully mutable :class:`PreparedRequest <PreparedRequest>` object,
380
containing the exact bytes that will be sent to the server.
382
Instances are generated from a :class:`Request <Request>` object, and
383
should not be instantiated manually; doing so may produce undesirable
389
>>> req = requests.Request('GET', 'https://httpbin.org/get')
390
>>> r = req.prepare()
392
<PreparedRequest [GET]>
394
>>> s = requests.Session()
401
headers: CaseInsensitiveDict[str | bytes]
402
_cookies: RequestsCookieJar | CookieJar | None
404
hooks: dict[str, list[_t.HookType]]
405
_body_position: int | object | None
407
def __init__(self) -> None:
408
#: HTTP verb to send to the server.
410
#: HTTP URL to send the request to.
412
#: dictionary of HTTP headers.
413
self.headers = None # type: ignore[assignment]
414
# The `CookieJar` used to create the Cookie header will be stored here
415
# after prepare_cookies is called
417
#: request body to send to the server.
419
#: dictionary of callback hooks, for internal usage.
420
self.hooks = default_hooks()
421
#: integer denoting starting position of a readable file-like body.
422
self._body_position = None
419
#: dictionary of callback hooks, for internal usage.
420
self.hooks = default_hooks()
421
#: integer denoting starting position of a readable file-like body.
422
self._body_position = None
426
method: str | None = None,
427
url: _t.UriType | None = None,
428
headers: Mapping[str, str | bytes] | None = None,
429
files: _t.FilesType = None,
430
data: _t.DataType = None,
431
params: _t.ParamsType = None,
432
auth: _t.AuthType = None,
433
cookies: RequestsCookieJar | CookieJar | dict[str, str] | None = None,
434
hooks: _t.HooksInputType | None = None,
435
json: _t.JsonType = None,
437
"""Prepares the entire request with the given parameters."""
439
url = cast("_t.UriType", url)
440
self.prepare_method(method)
441
self.prepare_url(url, params)
442
self.prepare_headers(headers)
443
self.prepare_cookies(cookies)
444
self.prepare_body(data, files, json)
445
self.prepare_auth(auth, url)
447
# Note that prepare_auth must be last to enable authentication schemes
448
# such as OAuth to work on a fully prepared request.
450
# This MUST go after prepare_auth. Authenticators could add a hook
451
self.prepare_hooks(hooks)
453
def __repr__(self) -> str:
454
return f"<PreparedRequest [{self.method}]>"
456
def copy(self) -> PreparedRequest:
457
p = PreparedRequest()
458
p.method = self.method
460
p.headers = self.headers.copy() if self.headers is not None else None # type: ignore[assignment]
461
p._cookies = _copy_cookie_jar(self._cookies)
464
p._body_position = self._body_position
461
p._cookies = _copy_cookie_jar(self._cookies)
464
p._body_position = self._body_position
467
def prepare_method(self, method: str | None) -> None:
468
"""Prepares the given HTTP method."""
470
if self.method is not None:
471
self.method = to_native_string(self.method.upper())
474
def _get_idna_encoded_host(host: str) -> str:
478
host = idna.encode(host, uts46=True).decode("utf-8")
479
except idna.IDNAError:
486
params: _t.ParamsType,
488
"""Prepares the given HTTP URL."""
489
#: Accept objects that have string representations.
490
#: We're unable to blindly call unicode/str functions
491
#: as this will include the bytestring indicator (b'')
493
#: https://github.com/psf/requests/pull/2238
494
if isinstance(url, bytes):
495
url = url.decode("utf8")
499
# Remove leading whitespaces from url
502
# Don't do any URL preparation for non-HTTP schemes like `mailto`,
503
# `data` etc to work around exceptions from `url_parse`, which
504
# handles RFC 3986 only.
505
if ":" in url and not url.lower().startswith("http"):
503
# `data` etc to work around exceptions from `url_parse`, which
504
# handles RFC 3986 only.
505
if ":" in url and not url.lower().startswith("http"):
509
# Support for unicode domain names and paths.
511
scheme, auth, host, port, path, query, fragment = parse_url(url)
512
except LocationParseError as e:
513
raise InvalidURL(*e.args)
517
f"Invalid URL {url!r}: No scheme supplied. "
518
f"Perhaps you meant https://{url}?"
522
raise InvalidURL(f"Invalid URL {url!r}: No host supplied")
524
# In general, we want to try IDNA encoding the hostname if the string contains
525
# non-ASCII characters. This allows users to automatically get the correct IDNA
526
# behaviour. For strings containing only ASCII characters, we need to also verify
527
# it doesn't start with a wildcard (*), before allowing the unencoded hostname.
528
if not unicode_is_ascii(host):
530
host = self._get_idna_encoded_host(host)
532
raise InvalidURL("URL has an invalid label.")
533
elif host.startswith(("*", ".")):
534
raise InvalidURL("URL has an invalid label.")
536
# Carefully reconstruct the network location
544
# Bare domains aren't valid URLs.
548
if isinstance(params, (str, bytes)):
549
params = to_native_string(params)
551
if params is not None:
552
enc_params = self._encode_params(params)
558
query = f"{query}&{enc_params}"
562
url = requote_uri(urlunparse((scheme, netloc, path, "", query, fragment)))
565
def prepare_headers(self, headers: Mapping[str, str | bytes] | None) -> None:
566
"""Prepares the given HTTP headers."""
568
self.headers = CaseInsensitiveDict()
570
for header in headers.items():
571
# Raise exception on invalid header value.
572
check_header_validity(header)
574
self.headers[to_native_string(name)] = value
577
self, data: _t.DataType, files: _t.FilesType, json: _t.JsonType = None
579
"""Prepares the given HTTP body data."""
581
# Check if file, fo, generator, iterator.
582
# If not, run through normal process.
582
# If not, run through normal process.
588
if not data and json is not None:
589
# urllib3 requires a bytes-like body. Python 2's json.dumps
590
# provides this natively, but Python 3 gives a Unicode string.
591
content_type = "application/json"
594
body = complexjson.dumps(json, allow_nan=False)
595
except ValueError as ve:
596
raise InvalidJSONError(ve, request=self)
598
if not isinstance(body, bytes):
599
body = body.encode("utf-8")
601
# data that proxies attributes to underlying objects needs hasattr
602
is_iterable = isinstance(data, Iterable) or hasattr(data, "__iter__")
603
if is_iterable and not isinstance(data, (str, bytes, list, tuple, Mapping)):
605
length = super_len(data)
606
except (TypeError, AttributeError, UnsupportedOperation):
611
if getattr(body, "tell", None) is not None:
612
# Record the current file position before reading.
613
# This will allow us to rewind a file in the event
616
self._body_position = body.tell() # type: ignore[union-attr] # guarded by getattr check
618
# This differentiates from None, allowing us to catch
619
# a failed `tell()` later when trying to rewind the body
620
self._body_position = object()
616
self._body_position = body.tell() # type: ignore[union-attr] # guarded by getattr check
618
# This differentiates from None, allowing us to catch
619
# a failed `tell()` later when trying to rewind the body
620
self._body_position = object()
623
raise NotImplementedError(
624
"Streamed bodies and files are mutually exclusive."
628
self.headers["Content-Length"] = builtin_str(length)
630
self.headers["Transfer-Encoding"] = "chunked"
632
# After is_stream filtering, remaining data is raw (not streamed)
633
raw_data = cast("_t.RawDataType | None", data)
635
# Multi-part file uploads.
637
(body, content_type) = self._encode_files(files, raw_data)
640
body = self._encode_params(raw_data)
641
if isinstance(data, basestring) or _t.has_read(data):
644
content_type = "application/x-www-form-urlencoded"
646
self.prepare_content_length(body)
648
# Add content-type if it wasn't explicitly provided.
649
if content_type and ("content-type" not in self.headers):
650
self.headers["Content-Type"] = content_type
646
self.prepare_content_length(body)
648
# Add content-type if it wasn't explicitly provided.
649
if content_type and ("content-type" not in self.headers):
650
self.headers["Content-Type"] = content_type
652
self.body = body # type: ignore[assignment] # body transforms from DataType to BodyType
654
def prepare_content_length(self, body: _t.BodyType) -> None:
655
"""Prepare Content-Length header based on request method and body"""
657
length = super_len(body)
659
# If length exists, set it. Otherwise, we fallback
660
# to Transfer-Encoding: chunked.
661
self.headers["Content-Length"] = builtin_str(length)
663
self.method not in ("GET", "HEAD")
664
and self.headers.get("Content-Length") is None
666
# Set Content-Length to 0 for methods that can have a body
667
# but don't provide one. (i.e. not GET or HEAD)
668
self.headers["Content-Length"] = "0"
673
url: _t.UriType = "",
675
"""Prepares the given HTTP auth data."""
677
# If no Auth is explicitly provided, extract it from the URL first.
679
url_auth = get_auth_from_url(cast(str, self.url))
680
auth = url_auth if any(url_auth) else None
677
# If no Auth is explicitly provided, extract it from the URL first.
679
url_auth = get_auth_from_url(cast(str, self.url))
680
auth = url_auth if any(url_auth) else None
683
if isinstance(auth, tuple) and len(auth) == 2: # type: ignore[arg-type] # pyright widens tuple from Callable in AuthType
684
# special-case basic HTTP auth
685
auth_handler = HTTPBasicAuth(*auth) # type: ignore[arg-type] # pyright widens tuple from Callable in AuthType
687
# TODO: can be fixed by flipping the conditionals
688
auth_handler = cast("Callable[..., PreparedRequest]", auth)
690
# Allow auth to make its changes.
691
r = auth_handler(self)
693
# Update self to reflect the auth changes.
694
self.__dict__.update(r.__dict__)
696
# Recompute Content-Length
697
self.prepare_content_length(self.body)
700
self, cookies: RequestsCookieJar | CookieJar | dict[str, str] | None
702
"""Prepares the given HTTP cookie data.
704
This function eventually generates a ``Cookie`` header from the
705
given cookies using cookielib. Due to cookielib's design, the header
706
will not be regenerated if it already exists, meaning this function
707
can only be called once for the life of the
708
:class:`PreparedRequest <PreparedRequest>` object. Any subsequent calls
709
to ``prepare_cookies`` will have no actual effect, unless the "Cookie"
710
header is removed beforehand.
712
if isinstance(cookies, cookielib.CookieJar):
713
self._cookies = cookies
715
self._cookies = cookiejar_from_dict(cookies)
712
if isinstance(cookies, cookielib.CookieJar):
713
self._cookies = cookies
715
self._cookies = cookiejar_from_dict(cookies)
717
cookies_jar = cast("CookieJar", self._cookies)
718
cookie_header = get_cookie_header(cookies_jar, self)
719
if cookie_header is not None:
720
self.headers["Cookie"] = cookie_header
722
def prepare_hooks(self, hooks: _t.HooksInputType | None) -> None:
723
"""Prepares the given hooks."""
724
# hooks can be passed as None to the prepare method and to this
725
# method. To prevent iterating over None, simply use an empty list
726
# if hooks is False-y
729
self.register_hook(event, hooks[event])
733
"""The :class:`Response <Response>` object, which contains a
734
server's response to an HTTP request.
737
_content: bytes | Literal[False] | None
738
_content_consumed: bool
739
_next: PreparedRequest | None
741
headers: CaseInsensitiveDict[str]
745
history: list[Response]
747
cookies: RequestsCookieJar
748
elapsed: datetime.timedelta
749
request: PreparedRequest
750
connection: HTTPAdapter
752
__attrs__: list[str] = [
765
def __init__(self) -> None:
766
self._content = False
767
self._content_consumed = False
770
#: Integer Code of responded HTTP Status, e.g. 404 or 200.
771
self.status_code = None # type: ignore[assignment]
773
#: Case-insensitive Dictionary of Response Headers.
774
#: For example, ``headers['content-encoding']`` will return the
775
#: value of a ``'Content-Encoding'`` response header.
776
self.headers = CaseInsensitiveDict()
778
#: File-like object representation of response (for advanced usage).
779
#: Use of ``raw`` requires that ``stream=True`` be set on the request.
780
#: This requirement does not apply for use internally to Requests.
783
#: Final URL location of Response.
784
self.url = None # type: ignore[assignment]
786
#: Encoding to decode with when accessing r.text.
789
#: A list of :class:`Response <Response>` objects from
790
#: the history of the Request. Any redirect responses will end
791
#: up here. The list is sorted from the oldest to the most recent request.
794
#: Textual reason of responded HTTP Status, e.g. "Not Found" or "OK".
795
self.reason = None # type: ignore[assignment]
797
#: A CookieJar of Cookies the server sent back.
798
self.cookies = cookiejar_from_dict({})
794
#: Textual reason of responded HTTP Status, e.g. "Not Found" or "OK".
795
self.reason = None # type: ignore[assignment]
797
#: A CookieJar of Cookies the server sent back.
798
self.cookies = cookiejar_from_dict({})
800
#: The amount of time elapsed between sending the request
801
#: and the arrival of the response (as a timedelta).
802
#: This property specifically measures the time taken between sending
803
#: the first byte of the request and finishing parsing the headers. It
804
#: is therefore unaffected by consuming the response content or the
805
#: value of the ``stream`` keyword argument.
806
self.elapsed = datetime.timedelta(0)
808
#: The :class:`PreparedRequest <PreparedRequest>` object to which this
810
self.request = None # type: ignore[assignment]
812
def __enter__(self) -> Self:
815
def __exit__(self, *args: Any) -> None:
818
def __getstate__(self) -> dict[str, Any]:
819
# Consume everything; accessing the content attribute makes
820
# sure the content has been fully read.
821
if not self._content_consumed:
824
return {attr: getattr(self, attr, None) for attr in self.__attrs__}
826
def __setstate__(self, state: dict[str, Any]) -> None:
827
for name, value in state.items():
828
setattr(self, name, value)
824
return {attr: getattr(self, attr, None) for attr in self.__attrs__}
826
def __setstate__(self, state: dict[str, Any]) -> None:
827
for name, value in state.items():
828
setattr(self, name, value)
830
# pickled objects do not have .raw
831
setattr(self, "_content_consumed", True)
832
setattr(self, "raw", None)
834
def __repr__(self) -> str:
835
return f"<Response [{self.status_code}]>"
837
def __bool__(self) -> bool:
838
"""Returns True if :attr:`status_code` is less than 400.
840
This attribute checks if the status code of the response is between
841
400 and 600 to see if there was a client error or a server error. If
842
the status code, is between 200 and 400, this will return True. This
843
is **not** a check to see if the response code is ``200 OK``.
847
def __nonzero__(self) -> bool:
848
"""Returns True if :attr:`status_code` is less than 400.
850
This attribute checks if the status code of the response is between
851
400 and 600 to see if there was a client error or a server error. If
852
the status code, is between 200 and 400, this will return True. This
853
is **not** a check to see if the response code is ``200 OK``.
857
def __iter__(self) -> Iterator[bytes]:
858
"""Allows you to use a response as an iterator."""
859
return self.iter_content(128)
857
def __iter__(self) -> Iterator[bytes]:
858
"""Allows you to use a response as an iterator."""
859
return self.iter_content(128)
862
def ok(self) -> bool:
863
"""Returns True if :attr:`status_code` is less than 400, False if not.
865
This attribute checks if the status code of the response is between
866
400 and 600 to see if there was a client error or a server error. If
867
the status code is between 200 and 400, this will return True. This
868
is **not** a check to see if the response code is ``200 OK``.
871
self.raise_for_status()
877
def is_redirect(self) -> bool:
878
"""True if this Response is a well-formed HTTP redirect that could have
879
been processed automatically (by :meth:`Session.resolve_redirects`).
881
return "location" in self.headers and self.status_code in REDIRECT_STATI
884
def is_permanent_redirect(self) -> bool:
885
"""True if this Response one of the permanent versions of redirect."""
886
return "location" in self.headers and self.status_code in (
887
codes.moved_permanently,
888
codes.permanent_redirect,
892
def next(self) -> PreparedRequest | None:
893
"""Returns a PreparedRequest for the next request in a redirect chain, if there is one."""
892
def next(self) -> PreparedRequest | None:
893
"""Returns a PreparedRequest for the next request in a redirect chain, if there is one."""
897
def apparent_encoding(self) -> str | None:
898
"""The apparent encoding, provided by the charset_normalizer or chardet libraries."""
899
if chardet is not None:
900
return chardet.detect(self.content)["encoding"]
902
# If no character detection library is available, we'll fall back
903
# to a standard Python utf-8 str.
908
self, chunk_size: int | None = 1, decode_unicode: Literal[False] = False
909
) -> Iterator[bytes]: ...
912
self, chunk_size: int | None = 1, *, decode_unicode: Literal[True]
913
) -> Iterator[str | bytes]: ...
915
self, chunk_size: int | None = 1, decode_unicode: bool = False
916
) -> Iterator[str | bytes]:
917
"""Iterates over the response data. When stream=True is set on the
918
request, this avoids reading the content at once into memory for
919
large responses. The chunk size is the number of bytes it should
920
read into memory. This is not necessarily the length of each item
921
returned as decoding can take place.
923
chunk_size must be of type int or None. A value of None will
924
function differently depending on the value of `stream`.
925
stream=True will read data as it arrives in whatever size the
926
chunks are received. If stream=False, data is returned as
923
chunk_size must be of type int or None. A value of None will
924
function differently depending on the value of `stream`.
925
stream=True will read data as it arrives in whatever size the
926
chunks are received. If stream=False, data is returned as
929
If decode_unicode is True, content will be decoded using encoding
930
information from the response. If no encoding information is available,
931
bytes will be returned. This can be bypassed by manually setting
932
`encoding` on the response.
935
def generate() -> Generator[bytes, None, None]:
936
# Special case for urllib3.
937
if hasattr(self.raw, "stream"):
939
yield from self.raw.stream(chunk_size, decode_content=True)
940
except ProtocolError as e:
941
raise ChunkedEncodingError(e)
942
except DecodeError as e:
943
raise ContentDecodingError(e)
944
except ReadTimeoutError as e:
945
raise ConnectionError(e)
946
except SSLError as e:
947
raise RequestsSSLError(e)
949
# Standard file-like object.
951
chunk = self.raw.read(chunk_size)
951
chunk = self.raw.read(chunk_size)
956
self._content_consumed = True
958
if self._content_consumed and isinstance(self._content, bool):
959
raise StreamConsumedError()
960
elif chunk_size is not None and not isinstance(
962
): # runtime guard for untyped callers
964
f"chunk_size must be an int, it is instead a {type(chunk_size)}."
967
if self._content_consumed:
968
# simulate reading small chunks of the content
969
content = cast(bytes, self._content)
970
chunks = iter_slices(content, chunk_size)
975
chunks = stream_decode_response_unicode(chunks, self)
982
chunk_size: int = ITER_CHUNK_SIZE,
983
decode_unicode: Literal[False] = False,
984
delimiter: bytes | None = None,
985
) -> Iterator[bytes]: ...
989
chunk_size: int = ITER_CHUNK_SIZE,
991
decode_unicode: Literal[True],
992
delimiter: str | bytes | None = None,
993
) -> Iterator[str | bytes]: ...
996
chunk_size: int = ITER_CHUNK_SIZE,
997
decode_unicode: bool = False,
998
delimiter: str | bytes | None = None,
999
) -> Iterator[str | bytes]:
1000
"""Iterates over the response data, one line at a time. When
1001
stream=True is set on the request, this avoids reading the
1002
content at once into memory for large responses.
998
delimiter: str | bytes | None = None,
999
) -> Iterator[str | bytes]:
1000
"""Iterates over the response data, one line at a time. When
1001
stream=True is set on the request, this avoids reading the
1002
content at once into memory for large responses.
1004
The decode_unicode param works the same as in `iter_content`, with the
1007
.. note:: This method is not reentrant safe.
1010
pending: str | bytes | None = None
1012
for chunk in self.iter_content(
1013
chunk_size=chunk_size, decode_unicode=decode_unicode
1015
if pending is not None:
1016
# TODO: remove cast after iter_lines rewrite
1017
chunk = cast("str | bytes", pending + chunk)
1020
lines = chunk.split(delimiter) # type: ignore[arg-type]
1022
lines = chunk.splitlines()
1024
if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]:
1025
pending = lines.pop()
1031
if pending is not None:
1035
def content(self) -> bytes:
1036
"""Content of the response, in bytes."""
1038
if self._content is False:
1039
# Read the contents.
1040
if self._content_consumed:
1041
raise RuntimeError("The content for this response was already consumed")
1038
if self._content is False:
1039
# Read the contents.
1040
if self._content_consumed:
1041
raise RuntimeError("The content for this response was already consumed")
1043
if self.status_code == 0 or self.raw is None:
1044
self._content = None
1046
self._content = b"".join(self.iter_content(CONTENT_CHUNK_SIZE)) or b""
1048
self._content_consumed = True
1049
# don't need to release the connection; that's been handled by urllib3
1050
# since we exhausted the data.
1051
return self._content # type: ignore[return-value]
1054
def text(self) -> str:
1055
"""Content of the response, in unicode.
1057
If Response.encoding is None, encoding will be guessed using
1058
``charset_normalizer`` or ``chardet``.
1060
The encoding of the response content is determined based solely on HTTP
1061
headers, following RFC 2616 to the letter. If you can take advantage of
1062
non-HTTP knowledge to make a better guess at the encoding, you should
1063
set ``r.encoding`` appropriately before accessing this property.
1066
# Try charset from content-type
1068
encoding = self.encoding
1070
if not self.content:
1073
# Fallback to auto-detected encoding.
1074
if self.encoding is None:
1075
encoding = self.apparent_encoding
1073
# Fallback to auto-detected encoding.
1074
if self.encoding is None:
1075
encoding = self.apparent_encoding
1077
# Decode unicode from given encoding.
1079
content = str(self.content, encoding or "utf-8", errors="replace")
1080
except (LookupError, TypeError):
1081
# A LookupError is raised if the encoding was not found which could
1082
# indicate a misspelling or similar mistake.
1084
# A TypeError can be raised if encoding is None
1086
# So we try blindly encoding.
1087
content = str(self.content, errors="replace")
1091
def json(self, **kwargs: Any) -> Any:
1092
r"""Decodes the JSON response body (if any) as a Python object.
1094
This may return a dictionary, list, etc. depending on what is in the response.
1096
:param \*\*kwargs: Optional arguments that ``json.loads`` takes.
1097
:raises requests.exceptions.JSONDecodeError: If the response body does not
1101
if not self.encoding and self.content and len(self.content) > 3:
1102
# No encoding set. JSON RFC 4627 section 3 states we should expect
1103
# UTF-8, -16 or -32. Detect which one to use; If the detection or
1104
# decoding fails, fall back to `self.text` (using charset_normalizer to make
1106
encoding = guess_json_utf(self.content)
1107
if encoding is not None:
1109
return complexjson.loads(self.content.decode(encoding), **kwargs)
1110
except UnicodeDecodeError:
1111
# Wrong UTF codec detected; usually because it's not UTF-8
1112
# but some other 8-bit codec. This is an RFC violation,
1113
# and the server didn't bother to tell us what codec *was*
1116
except JSONDecodeError as e:
1117
raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
1113
# and the server didn't bother to tell us what codec *was*
1116
except JSONDecodeError as e:
1117
raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
1120
return complexjson.loads(self.text, **kwargs)
1121
except JSONDecodeError as e:
1122
# Catch JSON-related errors and raise as requests.JSONDecodeError
1123
# This aliases json.JSONDecodeError and simplejson.JSONDecodeError
1124
raise RequestsJSONDecodeError(e.msg, e.doc, e.pos)
1127
def links(self) -> dict[str, dict[str, str]]:
1128
"""Returns the parsed header links of the response, if any."""
1130
header = self.headers.get("link")
1132
resolved_links: dict[str, dict[str, str]] = {}
1135
links = parse_header_links(header)
1138
key = link.get("rel") or link.get("url")
1140
resolved_links[key] = link
1142
return resolved_links
1144
def raise_for_status(self) -> None:
1145
"""Raises :class:`HTTPError`, if one occurred."""
1148
if isinstance(self.reason, bytes):
1149
# We attempt to decode utf-8 first because some servers
1150
# choose to localize their reason strings. If the string
1151
# isn't utf-8, we fall back to iso-8859-1 for all other
1152
# encodings. (See PR #3538)
1154
reason = self.reason.decode("utf-8")
1155
except UnicodeDecodeError:
1156
reason = self.reason.decode("iso-8859-1")
1158
reason = self.reason
1154
reason = self.reason.decode("utf-8")
1155
except UnicodeDecodeError:
1156
reason = self.reason.decode("iso-8859-1")
1158
reason = self.reason
1160
if 400 <= self.status_code < 500:
1162
f"{self.status_code} Client Error: {reason} for url: {self.url}"
1165
elif 500 <= self.status_code < 600:
1167
f"{self.status_code} Server Error: {reason} for url: {self.url}"
1171
raise HTTPError(http_error_msg, response=self)
1173
def close(self) -> None:
1174
"""Releases the connection back to the pool. Once this method has been
1175
called the underlying ``raw`` object must not be accessed again.
1177
*Note: Should not normally need to be called explicitly.*
1179
if not self._content_consumed:
1182
release_conn = getattr(self.raw, "release_conn", None)
1183
if release_conn is not None: