requests 的基本使用

安装 pip install requests

验证安装 import requests

GET请求

1. 抓取网页
2. 抓取二进制数据
3. 添加headers

post请求

#-*- coding:utf-8 -*-
import requests
data={'name':'jack','age':'22'}
r = requests.post("http://httpbin.org/post",data)
print(r.text)

响应

text 和 content 获取了响应的内容
requests.codes 内置状态码查询对象

文件上传

#-*- coding:utf-8 -*-
import requests
files = {'file':open('bg.png','rb')}
r =requests.post('http://httpbin.org/post',files = files)

print(r.text)

Cookie

import requests

r = requests.get('http://www.baidu.com')

print(r.cookies)

会话维持

利用session， 可以做到模拟同一个会话而不用担心Cookies的问题。通常用于模拟登录成功之后进行下一步操作

为了防止大规模爬取被封IP 设置代理，需要用到proxies

import requests

proxies ={
   "http":"http://user:password@10.10.1.10:3128/",
}

requests.get("https://www.taobao.com",proxies = proixes)

超时设置

timeout

身份验证

# -*- coding:utf-8 -*-

from urllib.request import HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler, build_opener
from urllib.error import URLError


username = 'username'
password = 'password'
url = 'http://localhost:5000/'

p = HTTPPasswordMgrWithDefaultRealm()
p.add_password(None, url, username, password)
auth_handler = HTTPBasicAuthHandler(p)
opener = build_opener(auth_handler)

try:
    result = opener.open(url)
    html = result.read().decode('utf-8')
    print(html)
except URLError as e:
    print(e.reason)

# -*- coding: utf-8 -*-
from functools import wraps

from flask import Flask, request, Response


def check_auth(username, password):
    """检查用户名和密码是否合法"""
    return username == 'username' and password == 'password'

def authenticate():
    """认证失败显示401错误"""
    return Response(
        '访问权限验证失败<br />\n请先登录',
        401,
        {
            'WWW-Authenticate': 'Basic realm="HTTP-Auth-Demo"'
        }
    )

def requires_auth(f):
    @wraps(f)
    def decorated(*args, **kwargs):
        auth = request.authorization
        if not auth or not check_auth(auth.username, auth.password):
            return authenticate()
        return f(*args, **kwargs)
    return decorated

app = Flask(__name__)

@app.route('/')
@requires_auth
def index_view():
    return 'Flask: ok'

if __name__ == '__main__':
    app.run(host='127.0.0.1', port=5000, debug=True)