requests库中呼吁的stream设为True就足以啊,requests库中呼吁的stream设为True就能够啦

室外下着中雨,作为独立程序员的自身逛着逛着发现一篇好东西,来自网易 你都用
Python
来做怎么样?
的率先个高亮答案。

露天下着中雨,作为单身程序员的本身逛着逛着发现一篇好东西,源于乐乎 你都用
Python
来做什么?
的率先个高亮答案。

到地点去看了看,地址都是当着的,得,赶紧初阶吧。

到地点去看了看,地址皆以精晓的,得,赶紧开首吧。

下载流式文件,requests库中呼吁的stream设为True就足以啊,亿万先生官方网站:,文书档案在此

下载流式文件,requests库中呼吁的stream设为True就能够啊,文书档案在此

先找贰个录制地址试验弹指间:

先找1个摄像地址试验瞬间:

# -*- coding: utf-8 -*-
import requests

def download_file(url, path):
    with requests.get(url, stream=True) as r:
        chunk_size = 1024
        content_size = int(r.headers['content-length'])
        print '下载开始'
        with open(path, "wb") as f:
            for chunk in r.iter_content(chunk_size=chunk_size):
                f.write(chunk)


if __name__ == '__main__':
    url = '就在原帖...'
    path = '想存哪都行'
    download_file(url, path)
# -*- coding: utf-8 -*-
import requests

def download_file(url, path):
    with requests.get(url, stream=True) as r:
        chunk_size = 1024
        content_size = int(r.headers['content-length'])
        print '下载开始'
        with open(path, "wb") as f:
            for chunk in r.iter_content(chunk_size=chunk_size):
                f.write(chunk)


if __name__ == '__main__':
    url = '就在原帖...'
    path = '想存哪都行'
    download_file(url, path)

备受当头一棒:

屡遭当头一棒:

AttributeError: __exit__
AttributeError: __exit__

这文书档案也会骗人的么!

这文书档案也会骗人的么!

观望是不曾落到实处上下文须求的__exit__格局。既然只是为了保障要让r最终close以释放连接池,那就应用contextlib的closing本性好了:

看来是未曾达成上下文需求的__exit__方法。既然只是为了保险要让r最终close以自由连接池,那就利用contextlib的closing性情好了:

# -*- coding: utf-8 -*-
import requests
from contextlib import closing

def download_file(url, path):
    with closing(requests.get(url, stream=True)) as r:
        chunk_size = 1024
        content_size = int(r.headers['content-length'])
        print '下载开始'
        with open(path, "wb") as f:
            for chunk in r.iter_content(chunk_size=chunk_size):
                f.write(chunk)
# -*- coding: utf-8 -*-
import requests
from contextlib import closing

def download_file(url, path):
    with closing(requests.get(url, stream=True)) as r:
        chunk_size = 1024
        content_size = int(r.headers['content-length'])
        print '下载开始'
        with open(path, "wb") as f:
            for chunk in r.iter_content(chunk_size=chunk_size):
                f.write(chunk)

次第寻常运行了,但是本身看着那文件,怎么大小不见变啊,到底是做到了有点了吗?照旧要让下好的内容及时存进硬盘,还是可以省点内部存款和储蓄器是或不是:

先后平常运作了,但是作者看着那文件,怎么大小不见变啊,到底是形成了多少了吗?仍然要让下好的始末及时存进硬盘,仍能省点内部存款和储蓄器是或不是:

# -*- coding: utf-8 -*-
import requests
from contextlib import closing
import os

def download_file(url, path):
    with closing(requests.get(url, stream=True)) as r:
        chunk_size = 1024
        content_size = int(r.headers['content-length'])
        print '下载开始'
        with open(path, "wb") as f:
            for chunk in r.iter_content(chunk_size=chunk_size):
                f.write(chunk)
                f.flush()
                os.fsync(f.fileno())
# -*- coding: utf-8 -*-
import requests
from contextlib import closing
import os

def download_file(url, path):
    with closing(requests.get(url, stream=True)) as r:
        chunk_size = 1024
        content_size = int(r.headers['content-length'])
        print '下载开始'
        with open(path, "wb") as f:
            for chunk in r.iter_content(chunk_size=chunk_size):
                f.write(chunk)
                f.flush()
                os.fsync(f.fileno())

文件以眼睛可知的快慢在增大,真心痛笔者的硬盘,依旧最终2回写入硬盘吧,程序中记个数就好了:

文件以肉眼可知的进程在增大,真可惜本人的硬盘,如故最终3回写入硬盘吧,程序中记个数就好了:

def download_file(url, path):
    with closing(requests.get(url, stream=True)) as r:
        chunk_size = 1024
        content_size = int(r.headers['content-length'])
        print '下载开始'
        with open(path, "wb") as f:
            n = 1
            for chunk in r.iter_content(chunk_size=chunk_size):
                loaded = n*1024.0/content_size
                f.write(chunk)
                print '已下载{0:%}'.format(loaded)
                n += 1
def download_file(url, path):
    with closing(requests.get(url, stream=True)) as r:
        chunk_size = 1024
        content_size = int(r.headers['content-length'])
        print '下载开始'
        with open(path, "wb") as f:
            n = 1
            for chunk in r.iter_content(chunk_size=chunk_size):
                loaded = n*1024.0/content_size
                f.write(chunk)
                print '已下载{0:%}'.format(loaded)
                n += 1

结果就很直观了:

结果就很直观了:

已下载2.579129%
已下载2.581255%
已下载2.583382%
已下载2.585508%
已下载2.579129%
已下载2.581255%
已下载2.583382%
已下载2.585508%

心怀远丹东想的自身怎么会只知足于这1个啊,写个类一起利用呢:

胸怀远东营想的自家怎么会只满足于那些吗,写个类一起使用啊:

# -*- coding: utf-8 -*-
import requests
from contextlib import closing
import time

def download_file(url, path):
    with closing(requests.get(url, stream=True)) as r:
        chunk_size = 1024*10
        content_size = int(r.headers['content-length'])
        print '下载开始'
        with open(path, "wb") as f:
            p = ProgressData(size = content_size, unit='Kb', block=chunk_size)
            for chunk in r.iter_content(chunk_size=chunk_size):
                f.write(chunk)
                p.output()


class ProgressData(object):

    def __init__(self, block,size, unit, file_name='', ):
        self.file_name = file_name
        self.block = block/1000.0
        self.size = size/1000.0
        self.unit = unit
        self.count = 0
        self.start = time.time()
    def output(self):
        self.end = time.time()
        self.count += 1
        speed = self.block/(self.end-self.start) if (self.end-self.start)>0 else 0
        self.start = time.time()
        loaded = self.count*self.block
        progress = round(loaded/self.size, 4)
        if loaded >= self.size:
            print u'%s下载完成\r\n'%self.file_name
        else:
            print u'{0}下载进度{1:.2f}{2}/{3:.2f}{4} 下载速度{5:.2%} {6:.2f}{7}/s'.\
                  format(self.file_name, loaded, self.unit,\
                  self.size, self.unit, progress, speed, self.unit)
            print '%50s'%('/'*int((1-progress)*50))
# -*- coding: utf-8 -*-
import requests
from contextlib import closing
import time

def download_file(url, path):
    with closing(requests.get(url, stream=True)) as r:
        chunk_size = 1024*10
        content_size = int(r.headers['content-length'])
        print '下载开始'
        with open(path, "wb") as f:
            p = ProgressData(size = content_size, unit='Kb', block=chunk_size)
            for chunk in r.iter_content(chunk_size=chunk_size):
                f.write(chunk)
                p.output()


class ProgressData(object):

    def __init__(self, block,size, unit, file_name='', ):
        self.file_name = file_name
        self.block = block/1000.0
        self.size = size/1000.0
        self.unit = unit
        self.count = 0
        self.start = time.time()
    def output(self):
        self.end = time.time()
        self.count += 1
        speed = self.block/(self.end-self.start) if (self.end-self.start)>0 else 0
        self.start = time.time()
        loaded = self.count*self.block
        progress = round(loaded/self.size, 4)
        if loaded >= self.size:
            print u'%s下载完成\r\n'%self.file_name
        else:
            print u'{0}下载进度{1:.2f}{2}/{3:.2f}{4} 下载速度{5:.2%} {6:.2f}{7}/s'.\
                  format(self.file_name, loaded, self.unit,\
                  self.size, self.unit, progress, speed, self.unit)
            print '%50s'%('/'*int((1-progress)*50))

运行:

运行:

下载开始
下载进度10.24Kb/120174.05Kb 0.01% 下载速度4.75Kb/s 
///////////////////////////////////////////////// 
下载进度20.48Kb/120174.05Kb 0.02% 下载速度32.93Kb/s 
/////////////////////////////////////////////////
下载开始
下载进度10.24Kb/120174.05Kb 0.01% 下载速度4.75Kb/s 
///////////////////////////////////////////////// 
下载进度20.48Kb/120174.05Kb 0.02% 下载速度32.93Kb/s 
/////////////////////////////////////////////////

看上去舒服多了。

看上去舒服多了。

上面要做的正是八线程同时下载了,主线程生产url放入队列,下载线程获取url:

上面要做的正是八线程同时下载了,主线程生产url放入队列,下载线程获取url:

# -*- coding: utf-8 -*-
import requests
from contextlib import closing
import time
import Queue
import hashlib
import threading
import os


def download_file(url, path):
    with closing(requests.get(url, stream=True)) as r:
        chunk_size = 1024*10
        content_size = int(r.headers['content-length'])
        if os.path.exists(path) and os.path.getsize(path)>=content_size:
            print '已下载'
            return
        print '下载开始'
        with open(path, "wb") as f:
            p = ProgressData(size = content_size, unit='Kb', block=chunk_size, file_name=path)
            for chunk in r.iter_content(chunk_size=chunk_size):
                f.write(chunk)
                p.output()


class ProgressData(object):

    def __init__(self, block,size, unit, file_name='', ):
        self.file_name = file_name
        self.block = block/1000.0
        self.size = size/1000.0
        self.unit = unit
        self.count = 0
        self.start = time.time()
    def output(self):
        self.end = time.time()
        self.count += 1
        speed = self.block/(self.end-self.start) if (self.end-self.start)>0 else 0
        self.start = time.time()
        loaded = self.count*self.block
        progress = round(loaded/self.size, 4)
        if loaded >= self.size:
            print u'%s下载完成\r\n'%self.file_name
        else:
            print u'{0}下载进度{1:.2f}{2}/{3:.2f}{4} {5:.2%} 下载速度{6:.2f}{7}/s'.\
                  format(self.file_name, loaded, self.unit,\
                  self.size, self.unit, progress, speed, self.unit)
            print '%50s'%('/'*int((1-progress)*50))


queue = Queue.Queue()


def run():
    while True:
        url = queue.get(timeout=100)
        if url is None:
            print u'全下完啦'
            break
        h = hashlib.md5()
        h.update(url)
        name = h.hexdigest()
        path = 'e:/download/' + name + '.mp4'
        download_file(url, path)


def get_url():
    queue.put(None)


if __name__ == '__main__':
    get_url()
    for i in xrange(4):
        t = threading.Thread(target=run)
        t.daemon = True
        t.start()
# -*- coding: utf-8 -*-
import requests
from contextlib import closing
import time
import Queue
import hashlib
import threading
import os


def download_file(url, path):
    with closing(requests.get(url, stream=True)) as r:
        chunk_size = 1024*10
        content_size = int(r.headers['content-length'])
        if os.path.exists(path) and os.path.getsize(path)>=content_size:
            print '已下载'
            return
        print '下载开始'
        with open(path, "wb") as f:
            p = ProgressData(size = content_size, unit='Kb', block=chunk_size, file_name=path)
            for chunk in r.iter_content(chunk_size=chunk_size):
                f.write(chunk)
                p.output()


class ProgressData(object):

    def __init__(self, block,size, unit, file_name='', ):
        self.file_name = file_name
        self.block = block/1000.0
        self.size = size/1000.0
        self.unit = unit
        self.count = 0
        self.start = time.time()
    def output(self):
        self.end = time.time()
        self.count += 1
        speed = self.block/(self.end-self.start) if (self.end-self.start)>0 else 0
        self.start = time.time()
        loaded = self.count*self.block
        progress = round(loaded/self.size, 4)
        if loaded >= self.size:
            print u'%s下载完成\r\n'%self.file_name
        else:
            print u'{0}下载进度{1:.2f}{2}/{3:.2f}{4} {5:.2%} 下载速度{6:.2f}{7}/s'.\
                  format(self.file_name, loaded, self.unit,\
                  self.size, self.unit, progress, speed, self.unit)
            print '%50s'%('/'*int((1-progress)*50))


queue = Queue.Queue()


def run():
    while True:
        url = queue.get(timeout=100)
        if url is None:
            print u'全下完啦'
            break
        h = hashlib.md5()
        h.update(url)
        name = h.hexdigest()
        path = 'e:/download/' + name + '.mp4'
        download_file(url, path)


def get_url():
    queue.put(None)


if __name__ == '__main__':
    get_url()
    for i in xrange(4):
        t = threading.Thread(target=run)
        t.daemon = True
        t.start()

加了再也下载的判断,至于怎么接连不断的生产url,诸位摸索吧,保重身体!

加了双重下载的判定,至于怎么接踵而至 蜂拥而至的生产url,诸位摸索吧,保重身体!

  

  

相关文章

网站地图xml地图