@@ -8,7 +8,7 @@ use reqwest_retry::RetryTransientMiddleware;
8
8
use std:: fs:: File ;
9
9
use std:: io:: BufRead ;
10
10
use std:: io:: BufReader ;
11
- use std:: path:: PathBuf ;
11
+ use std:: path:: { Path , PathBuf } ;
12
12
use std:: process;
13
13
use std:: sync:: Arc ;
14
14
use std:: time:: Duration ;
@@ -40,7 +40,7 @@ fn new_client(max_retries: usize) -> Result<ClientWithMiddleware, DownloadError>
40
40
pub async fn download_paths (
41
41
snapshot : & String ,
42
42
data_type : & str ,
43
- dst : & PathBuf ,
43
+ dst : & Path ,
44
44
) -> Result < ( ) , DownloadError > {
45
45
let paths = format ! ( "{}crawl-data/{}/{}.paths.gz" , BASE_URL , snapshot, data_type) ;
46
46
println ! ( "Downloading paths from: {}" , paths) ;
@@ -55,7 +55,7 @@ pub async fn download_paths(
55
55
56
56
let request = client. get ( url. as_str ( ) ) ;
57
57
58
- let mut dst = dst. clone ( ) ;
58
+ let mut dst = dst. to_path_buf ( ) ;
59
59
60
60
dst. push ( filename) ;
61
61
@@ -65,7 +65,7 @@ pub async fn download_paths(
65
65
let mut download = request. send ( ) . await ?;
66
66
67
67
while let Some ( chunk) = download. chunk ( ) . await ? {
68
- outfile. write ( & chunk) . await ?; // Write chunk to output file
68
+ outfile. write_all ( & chunk) . await ?; // Write chunk to output file
69
69
}
70
70
71
71
outfile. flush ( ) . await ?;
@@ -110,7 +110,7 @@ async fn download_task(
110
110
111
111
// Parse the filename from the given URL
112
112
let filename = if numbered {
113
- & format ! ( "{}{}" , number. to_string ( ) , ".txt.gz" )
113
+ & format ! ( "{}{}" , number, ".txt.gz" )
114
114
} else if files_only {
115
115
url. path_segments ( )
116
116
. and_then ( |segments| segments. last ( ) )
@@ -166,7 +166,7 @@ async fn download_task(
166
166
if progress {
167
167
progress_bar. inc ( chunk. len ( ) as u64 ) ; // Increase ProgressBar by chunk size
168
168
}
169
- outfile. write ( & chunk) . await ?; // Write chunk to output file
169
+ outfile. write_all ( & chunk) . await ?; // Write chunk to output file
170
170
}
171
171
172
172
if progress {
@@ -188,8 +188,8 @@ async fn download_task(
188
188
}
189
189
190
190
pub async fn download (
191
- paths : & PathBuf ,
192
- dst : & PathBuf ,
191
+ paths : & Path ,
192
+ dst : & Path ,
193
193
threads : usize ,
194
194
max_retries : usize ,
195
195
numbered : bool ,
@@ -253,11 +253,8 @@ pub async fn download(
253
253
let multibar = multibar. clone ( ) ;
254
254
let main_pb = main_pb. clone ( ) ;
255
255
let client = client. clone ( ) ;
256
- let dst = dst. clone ( ) ;
257
- let numbered = numbered. clone ( ) ;
258
- let files_only = files_only. clone ( ) ;
256
+ let dst = dst. to_path_buf ( ) ;
259
257
let semaphore = semaphore. clone ( ) ;
260
- let progress = progress. clone ( ) ;
261
258
set. spawn ( async move {
262
259
let _permit = semaphore. acquire ( ) . await ;
263
260
let res = download_task (
0 commit comments