"""画笔运动命令"""

t.left(degree)   #向左转
t.right(degree)  #向右转
t.pendown()      #放下画笔
t.penup()        #提起画笔
t.circle(半径，degree) #按多少半径和度数画圆，半径为负(正)圆心在画笔右边(左)
t.goto(x,y)           #将画笔移动到哪个坐标
t.forward(distance)   #向当前方向前进多少距离
t.backward(distance)  #向当前方向相反前进多少距离
home()            #设置当前画笔位置为原点，朝向东
seth(degree)            #设置当前朝向度数


"""画笔控制命令"""
t.fillcolor('color')  #绘制图形，填充颜色
t.color(color1,color2)  #同时设置pencolor、fillcolor
t.filling()        #返回是否处于填充状态
t.begin_fill()     #开始填充图形
t.end_fill()       #填充完成
t.hideturtle()     #隐藏画笔turtle形状
t.showturtle()     #显示画笔turtle形状
```  
### python基本知识
1. python程序由模块组成，一个模块对应python源文件，对应后缀.py
2. 模块由语句组成，程序依次执行
3. 一般4空格缩进，不强制空格数
4. `\`行连接符，使代码更加美观
5. python中一切皆对象，`对象本质就是：一个内存块，拥有特定值，支持特定类型的相关操作`
- 车位表示内存，id(标识identity)表示车位编号,车位停的什么车类型(type),详细车牌号表示值(value)
- id()内存地址，type()类型,print()值
- 赋给变量(对象引用)的是内存地址(对象)
6. 引用
- 变量位于 : 栈内存
- 对象位于 : 堆内存(id,type,value)
7. 标识符: 用于变量、函数、类、模块等名称
8. 赋值
- `链式赋值`用于对同一个对象进行多变量赋值 x=y=12 --> x=12,y=12
- `系列解包赋值`  a,b,c = 1,2,3 --> a=1,b=2,c=3 | a,b = b,a
- `常量`没有语法来限制改变常量值，可以改变
9. 循环性能优化
- 尽量减少循环内部不必要计算，往外层放
- 尽量使用局部变量查询较快
- 连接字符串使用join，不使用+(产生新的字符串对象)
- 列表操作(删除，插入)尽量在尾部，容易导致后面数据变动
10. zip()并行迭代
```python
names=['a','b','c']
ages=[1,2,3,4]
jobs=['好','事','情']
for i,k,j in zip(names,ages,jobs):
  print('{0}--{1}--{2}'.format(i,j,k))

#coding=gbk
from parse import parse_url
import json

class DoubanSpider:
    """豆瓣内容抓取"""
    def __init__(self):
        #最开始的入口
        self.temp_url = "https://movie.douban.com/j/search_subjects?type=tv&tag=%E7%BE%8E%E5%89%A7&sort=recommend&page_limit=20&page_start={}"       # {}大括号里对应字段传入值num的.format(num)
        
    def get_content_list(self,html_str):  #提取数据
        dict_data = json.loads(html_str)   #有callback就删除
        content_list = dict_data["subjects"]
        #total = dict_data["total"] 没有total的话自己设置
        return content_list   #,total
        
    def save_content_list(self,content_list):
        with open("douban.json","a") as f:
            for content in content_list:
                 f.write(json.dumps(content,ensure_ascii=False)) 
                 f.write("\n")
        print('保存成功')
        
    def run(self):  #实现主要逻辑
        num = 0
        #total = 100  初始化total,第二页更新时不使用
        while num < 100: #num < total+ page_limit 全部提取数据
            #1.start_url
            url = self.temp_url.format(num)  #对start进行复制
            print(url)
            #2.发送请求，获取响应
            html_str = parse_url(url)
            #3.提取数据
            content_list = self.get_content_list(html_str) #content,tatal = 
            #4.保存
            self.save_content_list(content_list)
            #5.构造下一页的url，循环2-5步
            num += 20
        
if __name__ == '__main__':
    douban = DoubanSpider()   #实例化
    douban.run()

#coding=gbk
from lxml import etree
import requests
import json

class QiubaiSpader:
    def __init__(self):
        self.url_temp = "https://www.qiushibaike.com/8hr/page/{}/"
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
        }
        
    def get_url_list(self):  #根据url地址规律构造url_list
        url_list = [self.url_temp.format(i) for i in range(1,14)]
        return url_list
        
    def parse_url(self,url): #发送请求获取响应
        print('now_parse:'+url)
        response = requests.get(url,headers=self.headers)
        return response.text.encode('GBK','ignore').decode('gbk')
        
    def get_content_list(self,html_str): #从字符串响应中提取数据
        html = etree.HTML(html_str) 
        content_list = [] 
        #1.分组
        div_list = html.xpath('//div[@id="content-left"]/div')
        for div in div_list:
            item = {}
            #获取的字符串[0]指的是去除两边换行\n,取除字符串
            item["author_name"] = div.xpath(".//h2/text()")[0].strip() if len(div.xpath(".//h2/text()"))>0 else None   #提取用户名
            item["author_img"] = div.xpath(".//img/@src")
            #内容
            item["content"] = div.xpath('.//div[@class="content"]/span/text()')  #<br>将字符串隔断
            item["content"] = [i.strip() for i in item["content"]]
            item["stats_vote"] = div.xpath('.//span[@class="stats-vote"]/i/text()')
            #好笑数
            item["stats_vote"] = item["stats_vote"][0] if len(item["stats_vote"])>0 else None
            item["stats-comments"] = div.xpath('.//span[@class="stats-comments"]//i/text()')
            #评论数
            item["stats-comments"] = item["stats-comments"][0] if len(item["stats-comments"])>0 else None
            item["content_img"] = div.xpath('.//div[@class="thumb"]//img/@src')
            #内容图片不完整加上获取
            item["content_img"] = 'https:' + item["content_img"][0] if len(item["content_img"])>0 else None
            content_list.append(item)
        return content_list   #返回json格式
        
    def save_content_list(self,content_list):#保存
        with open('qiubai.txt','a',encoding='utf-8') as f:
            for content in content_list:
            #转化为字符串
                f.write(json.dumps(content,ensure_ascii=False))
                f.write('\n')
        print("保存成功")
                
    def run(self):
        """实现主要逻辑"""
        #1.根据url地址规律来构造url列表 
        url_list = self.get_url_list()
        #2.发送请求获取响应
        for url in url_list:    #遍历一一爬取每个页面内容
            html_str = self.parse_url(url)
        #3.提取数据
        content_list = self.get_content_list(html_str)
        #4.保存
        self.save_content_list(content_list)

if __name__ == '__main__':
    qiubai = QiubaiSpader()
    qiubai.run()

"传智{}播客".format("1")
"传智{}播客".format(1)
"传智{}播客".format([1,2,3])
"传智{}播客{}".format({1,2,3},[1,2,3])

{i+10:i for i in range(10)} #{10:0,11:1..10:9}
{"a{}".format(i):10 for i in range(3)} # {"a0":10,"a1":10,"a2":10}

#coding=utf-8

import requests
from retrying import retry


headers = {
    "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1",
    }

@retry(stop_max_attempt_number=3) #装饰函数反复请求，三次请求都失败报错
def get_page(get_url):
    """get()方法请求内容"""
    print('*'*10)   #用于检测请求次数
    r = requests.get(get_url,headers=headers,timeout=10)
    return r.text.encode('GBK','ignore').decode('gbk')  #解决不支持'gbk'

def _get_page(get_url):
    try:
        html_str = get_page(get_url)
    except:
        html_str = 'Error'
    else:
        with open('baidu_get.text','w') as f:
            f.write(html_str[:500])      #正确执行写入文件
    return html_str

if __name__ == '__main__':
    
    get_url = 'https://www.baidu.com/'
    print(_get_page(get_url)[:500]) #请求前100字符串

#coding=gbk
import requests
from retrying import retry
import json

headers = {
    "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1",
    "Referer": "http://fanyi.baidu.com/?aldtype=16047",
    }
query_data = {
    "query": "人生苦短，我用python",             #此处可以用input输入字符串
    "from": "zh",
    "to": "en",
    }
    
@retry(stop_max_attempt_number=3) #连续请求三次，三次都失败报错
def get_page(url):
    """服务器请求"""
    print("*"*10)       #服务器请求检验
    r = requests.post(url,headers=headers,data=query_data,timeout=3)
    return r.content.decode()

def _get_page(url):
    try:
        html_str = get_page(url)
    except:
        html_str = None
    return html_str

if __name__ == '__main__':
    url = 'http://fanyi.baidu.com/basetrans'
    #print(_get_page(url)) #输出类似字典，但是json字符串
    page_dict = json.loads(_get_page(url))
    print(page_dict['trans'][0]['dst'])

#coding=gbk
import requests
from retrying import retry

#将请求成功的cookie信息放入headers，直接携带请求
headers = {
    "User_Agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1",
    "Cookie": "anonymid=jl6plze3-antvfl; _r01_=1; ln_uact=17318646019; ln_hurl=http://head.xiaonei.com/photos/0/0/men_main.gif; _de=71EA9E23AC1F39D48A6CD3B85E99E01A; depovince=SC; JSESSIONID=abcnrx7CjMin0-AScWtww; ick_login=9a8e1f1c-6cd8-4700-afa6-8cf5315d36c2; jebe_key=8be431ea-e18e-4feb-ab4b-0472222a3513%7C819b727b2164a924b41678741330f60a%7C1535114212567%7C1%7C1535782169180; _ga=GA1.2.609958396.1535782958; _gid=GA1.2.2069950083.1535782958; jebecookies=2f5e7612-0f24-4f28-a1ff-52b96074e414|||||; p=fddc828d0aeca708377ca86756cd44342; first_login_flag=1; t=84129fe73a07099108643adb89c95ad52; societyguester=84129fe73a07099108643adb89c95ad52; id=967724772; xnsid=6c333906; ver=7.0; loginfrom=null; wp_fold=0",
    }

@retry(stop_max_attempt_number=3)
def get_page(url):
    """请求服务器响应"""
    print('*'*10)
    response = requests.get(url,headers=headers,timeout=5)
    return response.content.decode()

def _get_page(url):
    try:
        html_str = get_page(url)
    except:
        html_str = None
    return html_str

if __name__ == '__main__':
    url = 'http://www.renren.com/967724772/profile'
    with open('renren2.txt','w',encoding='utf-8') as f:
        f.write(_get_page(url))

#coding=gbk
import requests
from retrying import retry

headers = {
    "User_Agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1",
    #"Referer": "http://www.renren.com/967724772"
    }

#cookie字符串
cookie="anonymid=jl6plze3-antvfl; _r01_=1; ln_uact=17318646019; ln_hurl=http://head.xiaonei.com/photos/0/0/men_main.gif; _de=71EA9E23AC1F39D48A6CD3B85E99E01A; depovince=SC; jebe_key=8be431ea-e18e-4feb-ab4b-0472222a3513%7C819b727b2164a924b41678741330f60a%7C1535114212567%7C1%7C1535782169180; _ga=GA1.2.609958396.1535782958; _gid=GA1.2.2069950083.1535782958; fenqi_user_city=36; jebecookies=35d2c64d-78b8-41ef-b5e4-4c887424f2ac|||||; JSESSIONID=abcJLhFJVpWjx73Ufvyww; ick_login=f74bfdc3-17bf-44a5-8227-33b2b0e79e89; p=f527cef24d899db1c7795ed77b5494772; first_login_flag=1; t=f634330753378f3c9eb40dd69dcd74c72; societyguester=f634330753378f3c9eb40dd69dcd74c72; id=967724772; xnsid=67ab8568; ver=7.0; loginfrom=null; wp_fold=0"

cookie_dict = {i.split("=")[0]:i.split("=")[-1]for i in cookie.split("; ")}   #字典推导式，将字符串转化为字典
#print(cookie_dict)   

@retry(stop_max_attempt_number=3)
def get_page(url):
    print('*'*10)
    r = requests.get(url,headers=headers,cookies=cookie_dict,timeout=3)
    return r.content.decode()

def _get_page(url):
    try:
        html_str = get_page(url)
    except:
        html_str = None
    return html_str

if __name__ == '__main__':
    url = "http://www.renren.com/967724772/profile"
    with open('renren2.txt','w') as f:
        f.write(_get_page(url))

#coding=gbk
import requests
from retrying import retry

headers = {
    "User_Agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1",
    #"Referer": "http://www.renren.com/967724772"
    }
#登录界面输入框内容和action网址
query_data = {
    "email":"17318646019",
    "password":"******",
    }

#实例化session
session = requests.session()

@retry(stop_max_attempt_number=3)
def _get_page(post_url,url):
    #先发送post请求，获取cookie,带上cookie请求登录后的页面
    session.post(post_url,headers=headers,data=query_data) #本地cookie保存到session
    #使用session请求登录后的页面
    response = session.get(url,headers=headers)
    return response.content.decode()
    
    
if __name__ == '__main__':
    post_url = "http://www.renren.com/PLogin.do"
    url = "http://www.renren.com/967724772/profile"
    with open('renren_session_cookie.txt','w',encoding='utf-8') as f:
        f.write(_get_page(post_url,url))

res_tr = r'<tr>(.*?)</tr>'      # re.M 改变'^'和'$'行为和 re.S改变'.'的行为
m_tr = re.findall(res_tr,language,re.S|re.M)

res_tr = r'<a .*?>(.*?)</a>'  
m_tr=  re.findall(res_tr, content, re.S|re.M)

urls=re.findall(r"<a.*?href=.*?<\/a>", content, re.I|re.S|re.M)
```  

- 获取<a href></a>中的URL

```python
res_url = r"(?<=href=\").+?(?=\")|(?<=href=\').+?(?=\')"  
link = re.findall(res_url ,  content, re.I|re.S|re.M)

title_pat = r'(?<=<title>).*?(?=</title>)'    
title_ex = re.compile(title_pat,re.M|re.S)    
title_obj = re.search(title_ex, content)  
title = title_obj.group()  
print title

title = re.findall(r'<title>(.*?)</title>', content)  
print title[0]

start = content.find(r'<table class="infobox vevent"') #起点记录查询位置    
end = content.find(r'</table>')        
infobox = language[start:end]    
print infobox   #得到start-end之间的结构体

if '<br />' in value:
       value = value.replace('<br />','')   #过滤该标签
       value = value.replace('\n',' ')      #换行空格替代 否则总换行

value = re.sub('<[^>]+>','', value)      #过滤HTML标签