Python: multiple files download by turn -
Displays the files downloaded and saved in the script loop (curl). But loop iterations are very fast, so there is no time to complete its operation in the download and saving tasks. Def run (self): Global DB C = DB Cursors (C) Execute ("" "Def run" "selection by command js_stones * stone_id limit 1" "") EC = EasyCurl (table), while (1) stone = c.fetchone () does, stone no ==: breaking img_fname = stone [2] Print img_fname url = "(" / "\ Var \\ dragon \\ \\ Jaston download Yogy_paccr \\" + Fname, progress = ec.textprogress)
this is a fraction,
# (url, filename) Tuples queue = qi. URL for url in url: url = url.strip () If no URL or URL then [= 0] == "#": file name = "doc_% 03d.dat"% (LAN (QEQ) ) + 1) queue.put ((URL, the file name)) # Czech args queue.queue loud, NUM_URLS "no claims URL" = len (queue.queue) num_conn = min (num_conn, NUM_URLS) 1 & Lt; = Num_conn & lt; A = 10000, "Invalid number of concurrent connections" Print "PycURL% s (against 0x% x compiled)"% (pycurl.version, pycurl.COMPILE_LIBCURL_VERSION_NUM) is printed "-----", NUM_URLS " Using the URL ", num_conn," connection ----- "class worker thread (threading thread): def __init __ (auto, line): threading. Thread .__ init __ (self) self.queue = queue def run (self): while 1: try: url, filename = self.queue.get_nowait (except queue.Empty: ("West Bengal" file name) curl = SystemExit pycurl.Curl () increased curl.setopt (pycurl.URL, URL) curl.setopt (pyc fp = open url.FOLLOWLOCATION, 1) curl.setopt (pycurl.MAXREDIRS, 5) curl.setopt (pycurl.CONNECTTIMEOUT try 30) curl.setopt (pycurl.TIMEOUT, 300) curl.setopt (pycurl.NOSIGNAL, 1) curl.setopt (pycurl.WRITEDATA, FP): except curl.perform (): import traceback traceback.print_exc (Start file = sys.stderr) sys.stderr.flush () curl.close () fp.close () sys.stdout.write ( ".") Sys.stdout.flush () # thread a bunch of threads = [] Dummy L In A range (num_conn): t = warkertread (queue) t.start () threads.append (t) # thread threads to end thread threads for all: thread .join ()
Comments
Post a Comment