Synopsis: 有时候 HTTP 请求失败了，可能是网络连接不稳定或者后端接口服务暂时未启动等原因，此时我们希望请求失败后能够进行一定次数的重试(retry)，从而最大限度保证请求能够最终成功。除了使用装饰器来实现以外，requests 模块中使用的 urllib3 模块默认自带重试功能

代码已上传到 https://github.com/wangy8961/python3-retry ，欢迎 star

假设我们已经按照 Flask Vue.js全栈开发｜第1章：创建第一个Flask RESTful API 构建了一个后端 API 接口，然后别人就可以通过 requests 模块来调用我们的接口：

import requests

from logger import logger


def main():
    url = 'http://127.0.0.1:5000/api/ping'  # 假设这是 Flask 后端接口服务，先关闭它

    try:
        resp = requests.get(url)
    except Exception as e:
        logger.error('Unable to establish connection with url({0}), exception: {1}'.format(url, e))
    else:
        resp_json = resp.json()
        if resp.status_code == 200:
            logger.info('Succeed to get response({0}) from url({1})'.format(resp_json, url))
        else:
            logger.error('Failed to get response from url({0}), status code: {1}'.format(url, resp.status_code))


if __name__ == '__main__':
    main()

python3 retry 01

但是，如果网络不稳定或者我们的 API 接口服务不小心被我们暂时关闭了时，就会抛出连接异常：

python3 retry 02

如果我们希望当 HTTP 请求失败时能实现 重试（retry） 功能，可以使用如下方法其中之一即可

1. 一般的用法

import time

import requests

from logger import logger


def main():
    url = 'http://127.0.0.1:5000/api/ping'  # 假设这是 Flask 后端接口服务，先关闭它

    while True:
        try:
            resp = requests.get(url)
        except Exception as e:
            logger.error('Unable to establish connection with url({0}), exception: {1}'.format(url, e))
            time.sleep(1)
        else:
            resp_json = resp.json()
            if resp.status_code == 200:
                logger.info('Succeed to get response({0}) from url({1})'.format(resp_json, url))
            else:
                logger.error('Failed to get response from url({0}), status code: {1}'.format(url, resp.status_code))
            break


if __name__ == '__main__':
    main()

python3 retry 03

2. 装饰器用法

参考 retry 自己写个简陋版本

2.1 同步版本

创建 sync_retry.py 文件：

from functools import wraps
import time

from logger import logger


def retry(tries=3, interval=0.3):
    """装饰器: 任何函数被调用时，如果抛出异常，允许重试"""
    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            count = 0
            while True:
                try:
                    result = func(*args, **kwargs)
                except Exception as e:
                    count += 1
                    if count > tries:  # 重试次数用完后，还是有异常时，只能往上抛出了
                        raise e
                    else:
                        logger.warning('Failed to call func {0}({4}, {5}), retrying in {1} seconds(total: {2}, retry: {3}). Exception: {6}'.format(func.__name__, interval, tries, count, args, kwargs, e))
                        time.sleep(interval)
                else:
                    return result
        return wrapper
    return decorator

测试：

import requests

from logger import logger
from sync_retry import retry


@retry()
def main(url):
    resp = requests.get(url)
    resp_json = resp.json()
    if resp.status_code == 200:
        logger.info('Succeed to get response({0}) from url({1})'.format(resp_json, url))
    else:
        logger.error('Failed to get response from url({0}), status code: {1}'.format(url, resp.status_code))


if __name__ == '__main__':
    url = 'http://127.0.0.1:5000/api/ping'  # 假设这是 Flask 后端接口服务，先关闭它

    try:
        main(url)
    except Exception as e:
        logger.error('Failed to get response from url({0}), exception: {1}'.format(url, e))

python3 retry 04

2.2 异步协程版本

创建 async_retry.py 文件：

import asyncio
from functools import wraps

from logger import logger
from tornado.simple_httpclient import HTTPTimeoutError


def async_retry(tries=3, interval=0.3):
    """装饰器: 原生协程函数被调用时，如果抛出异常，允许重试"""
    def decorator(func):
        @wraps(func)
        async def wrapper(*args, **kwargs):
            count = 0
            while True:
                try:
                    result = await func(*args, **kwargs)
                except Exception as e:
                    if isinstance(e, HTTPTimeoutError):  # tornado.simple_httpclient.HTTPTimeoutError 不用重试
                        raise e

                    count += 1
                    if count > tries:  # 重试次数用完后，还是有异常时，只能往上抛出了
                        raise e
                    else:
                        logger.warning('Failed to call func {0}({4}, {5}), retrying in {1} seconds(total: {2}, retry: {3}). Exception: {6}'.format(func.__name__, interval, tries, count, args, kwargs, e))
                        await asyncio.sleep(interval)
                else:
                    return result
        return wrapper
    return decorator

测试：

from tornado import httpclient, ioloop

from async_retry import async_retry
from logger import logger


@async_retry()
async def async_call_api(request, **kwargs):
    """支持异步重试功能"""
    httpclient.AsyncHTTPClient.configure(None, max_clients=1000)
    http_client = httpclient.AsyncHTTPClient()
    resp = await http_client.fetch(request, **kwargs)
    return resp


async def main():
    url = 'http://127.0.0.1:5000/api/ping'  # 假设这是 Flask 后端接口服务，先关闭它

    try:
        resp = await async_call_api(url, method='GET', connect_timeout=5, request_timeout=60)
    except Exception as e:
        logger.error('Failed to get response from url({0}), exception: {1}'.format(url, e))
    else:
        if resp.status_code == 200:
            logger.info('Succeed to get response({0}) from url({1})'.format(resp, url))
        else:
            logger.error('Failed to get response from url({0}), status code: {1}'.format(url, resp.status_code))


if __name__ == '__main__':
    io_loop = ioloop.IOLoop.current()
    io_loop.run_sync(main)

python3 retry 05

推荐功能更丰富的开源模块：

retrying：早已停止维护了
tenacity： fork 自 retrying，目前一直在更新，推荐使用
backoff

3. urllib3 自带 Retry

如果你只是想在使用 requests 模块时重试，完全可以用它自带的重试功能即可。创建 requests_retry.py 文件：

import requests
from requests.adapters import HTTPAdapter
from urllib3.util import Retry


def requests_retry_session(
    retries=3,
    backoff_factor=0.3,  # 重试 3 次时，总耗时 1.8 = 0 + 0.6 + 1.2 秒
    status_forcelist=(500, 502, 504),
    session=None,
):
    session = session or requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
        method_whitelist=frozenset(['GET', 'POST', 'PUT', 'DELETE', 'HEAD', 'OPTIONS'])  # urllib3 默认对除 GET 以外的方法，不设置自动重试功能，所以要主动添加白名单
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)

    return session

测试：

from logger import logger
from requests_retry import requests_retry_session


def main():
    url = 'http://127.0.0.1:5000/api/ping'  # 假设这是 Flask 后端接口服务，先关闭它
    # url = 'https://httpbin.org/status/502'  # 模拟返回 502 响应码

    try:
        resp = requests_retry_session().get(url)
    except Exception as e:
        logger.error('Unable to establish connection with url({0}), exception: {1}'.format(url, e))
    else:
        resp_json = resp.json()
        if resp.status_code == 200:
            logger.info('Succeed to get response({0}) from url({1})'.format(resp_json, url))
        else:
            logger.error('Failed to get response from url({0}), status code: {1}'.format(url, resp.status_code))


if __name__ == '__main__':
    main()