1 module dast.net.curlapi; 2 3 import dast.net.dlfcn; 4 import etc.c.curl : CurlGlobal; 5 import std.exception : enforce; 6 7 /++ 8 Exception thrown on errors in std.net.curl functions. 9 +/ 10 class CurlException : Exception { 11 /++ 12 Params: 13 msg = The message for the exception. 14 file = The file where the exception occurred. 15 line = The line number where the exception occurred. 16 next = The previous exception in the chain of exceptions, if any. 17 +/ 18 @safe pure nothrow 19 this(string msg, 20 string file = __FILE__, 21 size_t line = __LINE__, 22 Throwable next = null) { 23 super(msg, file, line, next); 24 } 25 } 26 27 /++ 28 Exception thrown on timeout errors in std.net.curl functions. 29 +/ 30 class CurlTimeoutException : CurlException { 31 /++ 32 Params: 33 msg = The message for the exception. 34 file = The file where the exception occurred. 35 line = The line number where the exception occurred. 36 next = The previous exception in the chain of exceptions, if any. 37 +/ 38 @safe pure nothrow 39 this(string msg, 40 string file = __FILE__, 41 size_t line = __LINE__, 42 Throwable next = null) { 43 super(msg, file, line, next); 44 } 45 } 46 47 enum CurlInf : CurlInfo { 48 contentlengthdownload = cast(CurlInfo)6_291_471, 49 contentlengthupload = cast(CurlInfo)6_291_472, 50 } 51 52 struct CurlAPI { 53 import etc.c.curl : curl_version_info_data, 54 CURL, CURLcode, CURLINFO, CURLoption, CURLversion, curl_slist; 55 56 extern (C) nothrow @nogc: 57 import core.stdc.config : c_long; 58 59 CURLcode function(c_long flags) global_init; 60 void function() global_cleanup; 61 curl_version_info_data* function(CURLversion) version_info; 62 CURL* function() easy_init; 63 CURLcode function(CURL* curl, CURLoption option, ...) easy_setopt; 64 CURLcode function(CURL* curl) easy_perform; 65 CURLcode function(CURL* curl, CURLINFO info, ...) easy_getinfo; 66 CURL* function(CURL* curl) easy_duphandle; 67 char* function(CURLcode) easy_strerror; 68 CURLcode function(CURL* handle, int bitmask) easy_pause; 69 void function(CURL* curl) easy_cleanup; 70 curl_slist* function(curl_slist*, char*) slist_append; 71 void function(curl_slist*) slist_free_all; 72 void function(void*) free; 73 char* function(CURL* curl, scope const(char)* s, int length) easy_escape; 74 char* function(CURL* curl, scope const(char)* s, int length, int* outlength) easy_unescape; 75 76 CURL* function() multi_init; 77 CURLMcode function(void* multi_handle, void* extra_fds, uint extra_nfds, int timeout_ms, int* ret) multi_wait; 78 CURLMcode function(void* multi_handle, CURLMoption option, ...) multi_setopt; 79 CURLMcode function(void* multi_handle, CURLMcode* running_handles) multi_perform; 80 CURLMsg* function(void* multi_handle, int* msg_left) multi_info_read; 81 CURLMcode function(void* multi_handle, CURL* easy_handle) multi_add_handle; 82 CURLMcode function(void* multi_handle, CURL* easy_handle) multi_remove_handle; 83 void function(void* multi_handle) multi_cleanup; 84 } 85 86 private __gshared { 87 CurlAPI _api; 88 void* _handle; 89 } 90 91 @property ref curlAPI() { 92 import std.concurrency : initOnce; 93 94 initOnce!_handle(loadAPI()); 95 return _api; 96 } 97 98 // TODO: use AliasSeq 99 version (LibcurlPath) { 100 import std.string : strip; 101 102 private immutable names = [strip(import("LibcurlPathFile"))]; 103 } else version (OSX) 104 private immutable names = ["libcurl.4.dylib"]; 105 else version (Posix) { 106 private immutable names = [ 107 "libcurl.so", "libcurl.so.4", 108 "libcurl-gnutls.so.4", "libcurl-nss.so.4", "libcurl.so.3" 109 ]; 110 } else version (Windows) 111 private immutable names = ["libcurl.dll", "curl.dll"]; 112 113 void* loadAPI() { 114 import core.stdc.stdlib : atexit; 115 import std.format : format; 116 117 void* handle; 118 foreach (name; names) { 119 handle = dlopen(name.ptr, RTLD_LAZY); 120 if (handle !is null) 121 break; 122 } 123 124 enforce!CurlException(handle !is null, "Failed to load curl, tried %(%s, %).".format(names)); 125 126 foreach (i, ref f; _api.tupleof) { 127 enum name = __traits(identifier, _api.tupleof[i]); 128 auto p = enforce!CurlException(dlsym(handle, "curl_" ~ name), 129 "Couldn't load curl_" ~ name ~ " from libcurl."); 130 f = cast(typeof(f))p; 131 } 132 133 enforce!CurlException(!_api.global_init(CurlGlobal.all), 134 "Failed to initialize libcurl"); 135 136 atexit(&cleanup); 137 138 return handle; 139 } 140 141 extern (C) void cleanup() { 142 if (_handle is null) 143 return; 144 _api.global_cleanup(); 145 146 dlclose(_handle); 147 _api = CurlAPI.init; 148 _handle = null; 149 } 150 151 /** 152 Wrapper to provide a better interface to libcurl than using the plain C API. 153 154 Warning: This struct uses interior pointers for callbacks. Only allocate it 155 on the stack if you never move or copy it. This also means passing by reference 156 when passing Curl to other functions. Otherwise always allocate on 157 the heap. 158 */ 159 struct Curl { 160 import etc.c.curl; 161 162 alias OutData = void[]; 163 alias InData = ubyte[]; 164 165 private { 166 alias curl = curlAPI; 167 168 // A handle should not be used by two threads simultaneously 169 CURL* ch; 170 uint _rc; 171 172 // May also return `CURL_READFUNC_ABORT` or `CURL_READFUNC_PAUSE` 173 size_t delegate(OutData) _onSend; 174 size_t delegate(InData) _onReceive; 175 void delegate(in char[]) _onReceiveHeader; 176 CurlSeek delegate(long, CurlSeekPos) _onSeek; 177 int delegate(curl_socket_t, CurlSockType) _onSocketOption; 178 int delegate(size_t dltotal, size_t dlnow, 179 size_t ultotal, size_t ulnow) _onProgress; 180 } 181 182 //alias requestPause = CurlReadFunc.pause; 183 //alias requestAbort = CurlReadFunc.abort; 184 185 this(CURL* h) { 186 if (h) 187 ch = h; 188 else { 189 ch = curl.easy_init(); 190 enforce!CurlException(ch, "Curl instance couldn't be initialized"); 191 set(CurlOption.nosignal, 1); 192 } 193 _rc = 1; 194 } 195 196 @disable this(); 197 198 this(this) { 199 ++_rc; 200 } 201 202 ~this() { 203 if (ch) 204 close(); 205 } 206 207 @property handle() => ch; 208 209 /// 210 @property bool stopped() const => ch is null; 211 212 /** 213 Duplicate this handle. 214 215 The new handle will have all options set as the one it was duplicated 216 from. An exception to this is that all options that cannot be shared 217 across threads are reset thereby making it safe to use the duplicate 218 in a new thread. 219 */ 220 Curl dup() { 221 import std.meta : AliasSeq; 222 223 Curl copy = Curl(curl.easy_duphandle(ch)); 224 225 with (CurlOption) { 226 foreach (opt; AliasSeq!(file, writefunction, writeheader, 227 headerfunction, infile, readfunction, ioctldata, ioctlfunction, 228 seekdata, seekfunction, sockoptdata, sockoptfunction, 229 opensocketdata, opensocketfunction, progressdata, 230 progressfunction, debugdata, debugfunction, interleavedata, 231 interleavefunction, chunk_data, chunk_bgn_function, 232 chunk_end_function, fnmatch_data, fnmatch_function, cookiejar, postfields)) 233 copy.clear(opt); 234 235 // The options are only supported by libcurl when it has been built 236 // against certain versions of OpenSSL - if your libcurl uses an old 237 // OpenSSL, or uses an entirely different SSL engine, attempting to 238 // clear these normally will raise an exception 239 copy.tryClear(ssl_ctx_function); 240 copy.tryClear(ssh_keydata); 241 } 242 243 // Enable for curl version > 7.21.7 244 static if (LIBCURL_VERSION_MAJOR >= 7 && 245 LIBCURL_VERSION_MINOR >= 21 && 246 LIBCURL_VERSION_PATCH >= 7) { 247 copy.clear(CurlOption.closesocketdata); 248 copy.clear(CurlOption.closesocketfunction); 249 } 250 251 copy.set(CurlOption.nosignal, 1); 252 253 // copy.clear(CurlOption.ssl_ctx_data); Let ssl function be shared 254 // copy.clear(CurlOption.ssh_keyfunction); Let key function be shared 255 256 /* 257 Allow sharing of conv functions 258 copy.clear(CurlOption.conv_to_network_function); 259 copy.clear(CurlOption.conv_from_network_function); 260 copy.clear(CurlOption.conv_from_utf8_function); 261 */ 262 263 return copy; 264 } 265 266 /** 267 Stop and invalidate this curl instance. 268 Warning: Do not call this from inside a callback handler e.g. `onReceive`. 269 */ 270 void close() { 271 throwOnStopped(); 272 if (--_rc == 0) 273 curl.easy_cleanup(ch); 274 ch = null; 275 } 276 277 /** 278 Pausing and continuing transfers. 279 */ 280 void pause(bool sendingPaused, bool receivingPaused) { 281 throwOnStopped(); 282 check(curl.easy_pause(ch, 283 (sendingPaused ? CurlPause.send_cont 284 : CurlPause.send) | 285 (receivingPaused ? CurlPause.recv_cont : CurlPause.recv))); 286 } 287 288 /** 289 Set a string curl option. 290 Params: 291 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation 292 value = The string 293 */ 294 void set(CurlOption option, const(char)[] value) { 295 import std.internal.cstring : tempCString; 296 297 throwOnStopped(); 298 check(curl.easy_setopt(ch, option, value.tempCString().buffPtr)); 299 } 300 301 /** 302 Set a long curl option. 303 Params: 304 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation 305 value = The long 306 */ 307 void set(CurlOption option, long value) { 308 throwOnStopped(); 309 check(curl.easy_setopt(ch, option, value)); 310 } 311 312 /** 313 Set a void* curl option. 314 Params: 315 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation 316 value = The pointer 317 */ 318 void set(CurlOption option, void* value) { 319 throwOnStopped(); 320 check(curl.easy_setopt(ch, option, value)); 321 } 322 323 /** 324 Clear a pointer option. 325 Params: 326 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation 327 */ 328 void clear(CurlOption option) { 329 check(tryClear(option)); 330 } 331 332 /** 333 Clear a pointer option. Does not raise an exception if the underlying 334 libcurl does not support the option. Use sparingly. 335 Params: 336 option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation 337 */ 338 CURLcode tryClear(CurlOption option) { 339 throwOnStopped(); 340 return curl.easy_setopt(ch, option, null); 341 } 342 343 /** 344 perform the curl request by doing the HTTP,FTP etc. as it has 345 been setup beforehand. 346 347 Params: 348 throwOnError = whether to throw an exception or return a CURLcode on error 349 */ 350 CURLcode perform(bool throwOnError = true) { 351 throwOnStopped(); 352 const code = curl.easy_perform(ch); 353 if (throwOnError) 354 check(code); 355 return code; 356 } 357 358 CURLcode tryGet(CurlInfo info, ref long val) 359 => curl.easy_getinfo(ch, info, &val); 360 361 long get(CurlInfo info) { 362 long val; 363 return curl.easy_getinfo(ch, info, &val) ? -1 : val; 364 } 365 366 /** 367 * The event handler that receives incoming data. 368 * 369 * Params: 370 * callback = the callback that receives the `ubyte[]` data. 371 * Be sure to copy the incoming data and not store 372 * a slice. 373 * 374 * Returns: 375 * The callback returns the incoming bytes read. If not the entire array is 376 * the request will abort. 377 * The special value HTTP.pauseRequest can be returned in order to pause the 378 * current request. 379 * 380 * Example: 381 * --- 382 * import std.net.curl, std.stdio, std.conv; 383 * Curl curl; 384 * curl.initialize(); 385 * curl.set(CurlOption.url, "http://dlang.org"); 386 * curl.onReceive = (ubyte[] data) { writeln("Got data", to!(const(char)[])(data)); return data.length;}; 387 * curl.perform(); 388 * --- 389 */ 390 @property void onReceive(typeof(_onReceive) callback) { 391 _onReceive = (InData id) { 392 throwOnStopped("Receive callback called on cleaned up Curl instance"); 393 return callback(id); 394 }; 395 set(CurlOption.file, &this); 396 set(CurlOption.writefunction, &_receive); 397 } 398 399 /** 400 * The event handler that receives incoming headers for protocols 401 * that uses headers. 402 * 403 * Params: 404 * callback = the callback that receives the header string. 405 * Make sure the callback copies the incoming params if 406 * it needs to store it because they are references into 407 * the backend and may very likely change. 408 * 409 * Example: 410 * --- 411 * import std.net.curl, std.stdio; 412 * Curl curl; 413 * curl.initialize(); 414 * curl.set(CurlOption.url, "http://dlang.org"); 415 * curl.onReceiveHeader = (in char[] header) { writeln(header); }; 416 * curl.perform(); 417 * --- 418 */ 419 @property void onReceiveHeader(void delegate(in char[]) callback) { 420 _onReceiveHeader = (in char[] od) { 421 throwOnStopped("Receive header callback called on " ~ 422 "cleaned up Curl instance"); 423 callback(od); 424 }; 425 set(CurlOption.writeheader, &this); 426 set(CurlOption.headerfunction, &_receiveHeader); 427 } 428 429 /** 430 * The event handler that gets called when data is needed for sending. 431 * 432 * Params: 433 * callback = the callback that has a `void[]` buffer to be filled 434 * 435 * Returns: 436 * The callback returns the number of elements in the buffer that have been 437 * filled and are ready to send. 438 * The special value `Curl.abortRequest` can be returned in 439 * order to abort the current request. 440 * The special value `Curl.pauseRequest` can be returned in order to 441 * pause the current request. 442 * 443 * Example: 444 * --- 445 * import std.net.curl; 446 * Curl curl; 447 * curl.initialize(); 448 * curl.set(CurlOption.url, "http://dlang.org"); 449 * 450 * string msg = "Hello world"; 451 * curl.onSend = (void[] data) 452 * { 453 * auto m = cast(void[]) msg; 454 * size_t length = m.length > data.length ? data.length : m.length; 455 * if (length == 0) return 0; 456 * data[0 .. length] = m[0 .. length]; 457 * msg = msg[length..$]; 458 * return length; 459 * }; 460 * curl.perform(); 461 * --- 462 */ 463 @property void onSend(typeof(_onSend) callback) { 464 _onSend = (OutData od) { 465 throwOnStopped("Send callback called on cleaned up Curl instance"); 466 return callback(od); 467 }; 468 set(CurlOption.infile, &this); 469 set(CurlOption.readfunction, &_send); 470 } 471 472 /** 473 * The event handler that gets called when the curl backend needs to seek 474 * the data to be sent. 475 * 476 * Params: 477 * callback = the callback that receives a seek offset and a seek position 478 * $(REF CurlSeekPos, etc,c,curl) 479 * 480 * Returns: 481 * The callback returns the success state of the seeking 482 * $(REF CurlSeek, etc,c,curl) 483 * 484 * Example: 485 * --- 486 * import std.net.curl; 487 * Curl curl; 488 * curl.initialize(); 489 * curl.set(CurlOption.url, "http://dlang.org"); 490 * curl.onSeek = (long p, CurlSeekPos sp) 491 * { 492 * return CurlSeek.cantseek; 493 * }; 494 * curl.perform(); 495 * --- 496 */ 497 @property void onSeek(typeof(_onSeek) callback) { 498 _onSeek = (long ofs, CurlSeekPos sp) { 499 throwOnStopped("Seek callback called on cleaned up Curl instance"); 500 return callback(ofs, sp); 501 }; 502 set(CurlOption.seekdata, &this); 503 set(CurlOption.seekfunction, &_seek); 504 } 505 506 /** 507 * The event handler that gets called when the net socket has been created 508 * but a `connect()` call has not yet been done. This makes it possible to set 509 * misc. socket options. 510 * 511 * Params: 512 * callback = the callback that receives the socket and socket type 513 * $(REF CurlSockType, etc,c,curl) 514 * 515 * Returns: 516 * Return 0 from the callback to signal success, return 1 to signal error 517 * and make curl close the socket 518 * 519 * Example: 520 * --- 521 * import std.net.curl; 522 * Curl curl; 523 * curl.initialize(); 524 * curl.set(CurlOption.url, "http://dlang.org"); 525 * curl.onSocketOption = delegate int(curl_socket_t s, CurlSockType t) { /+ do stuff +/ }; 526 * curl.perform(); 527 * --- 528 */ 529 @property void onSocketOption(typeof(_onSocketOption) callback) { 530 _onSocketOption = (curl_socket_t sock, CurlSockType st) { 531 throwOnStopped("Socket option callback called on " ~ 532 "cleaned up Curl instance"); 533 return callback(sock, st); 534 }; 535 set(CurlOption.sockoptdata, &this); 536 set(CurlOption.sockoptfunction, &_socketOption); 537 } 538 539 /** 540 * The event handler that gets called to inform of upload/download progress. 541 * 542 * Params: 543 * callback = the callback that receives the (total bytes to download, 544 * currently downloaded bytes, total bytes to upload, currently uploaded 545 * bytes). 546 * 547 * Returns: 548 * Return 0 from the callback to signal success, return non-zero to abort 549 * transfer 550 * 551 * Example: 552 * --- 553 * import std.net.curl, std.stdio; 554 * Curl curl; 555 * curl.initialize(); 556 * curl.set(CurlOption.url, "http://dlang.org"); 557 * curl.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow) 558 * { 559 * writeln("Progress: downloaded bytes ", dlnow, " of ", dltotal); 560 * writeln("Progress: uploaded bytes ", ulnow, " of ", ultotal); 561 * return 0; 562 * }; 563 * curl.perform(); 564 * --- 565 */ 566 @property void onProgress(typeof(_onProgress) callback) { 567 _onProgress = (size_t dlt, size_t dln, size_t ult, size_t uln) { 568 throwOnStopped("Progress callback called on cleaned " ~ 569 "up Curl instance"); 570 return callback(dlt, dln, ult, uln); 571 }; 572 set(CurlOption.noprogress, 0); 573 set(CurlOption.progressdata, &this); 574 set(CurlOption.progressfunction, &_progress); 575 } 576 577 private: 578 579 void check(CURLcode code) { 580 enforce!CurlTimeoutException(code != CurlError.operation_timedout, 581 errorString(code)); 582 583 enforce!CurlException(code == CurlError.ok, 584 errorString(code)); 585 } 586 587 string errorString(CURLcode code) { 588 import std.string : fromStringz; 589 import std.format : format; 590 591 const msgZ = curl.easy_strerror(code); 592 // doing the following (instead of just using std.conv.to!string) avoids 1 allocation 593 return "%s on handle %s".format(fromStringz(msgZ), ch); 594 } 595 596 void throwOnStopped(string message = null) { 597 enum def = "Curl instance called after being cleaned up"; 598 enforce!CurlException(!stopped, 599 message == null ? def : message); 600 } 601 } 602 603 import etc.c.curl; 604 605 private extern (C): 606 // Internal C callbacks to register with libcurl 607 size_t _receive(const char* str, 608 size_t size, size_t nmemb, void* ptr) { 609 auto b = cast(Curl*)ptr; 610 if (b._onReceive != null) 611 return b._onReceive(cast(Curl.InData)str[0 .. size * nmemb]); 612 return size * nmemb; 613 } 614 615 size_t _receiveHeader(const char* str, 616 size_t size, size_t nmemb, void* ptr) { 617 import std.string : chomp; 618 619 auto b = cast(Curl*)ptr; 620 const s = str[0 .. size * nmemb].chomp(); 621 if (b._onReceiveHeader != null) 622 b._onReceiveHeader(s); 623 624 return size * nmemb; 625 } 626 627 size_t _send(char* str, size_t size, size_t nmemb, void* ptr) { 628 Curl* b = cast(Curl*)ptr; 629 auto a = cast(void[])str[0 .. size * nmemb]; 630 if (b._onSend == null) 631 return 0; 632 return b._onSend(a); 633 } 634 635 int _seek(void* ptr, curl_off_t offset, int origin) { 636 auto b = cast(Curl*)ptr; 637 if (b._onSeek == null) 638 return CurlSeek.cantseek; 639 640 // origin: CurlSeekPos.set/current/end 641 // return: CurlSeek.ok/fail/cantseek 642 return b._onSeek(cast(long)offset, cast(CurlSeekPos)origin); 643 } 644 645 int _socketOption(void* ptr, 646 curl_socket_t curlfd, curlsocktype purpose) { 647 auto b = cast(Curl*)ptr; 648 if (b._onSocketOption == null) 649 return 0; 650 651 // return: 0 ok, 1 fail 652 return b._onSocketOption(curlfd, cast(CurlSockType)purpose); 653 } 654 655 int _progress(void* ptr, 656 double dltotal, double dlnow, 657 double ultotal, double ulnow) { 658 auto b = cast(Curl*)ptr; 659 if (b._onProgress == null) 660 return 0; 661 662 // return: 0 ok, 1 fail 663 return b._onProgress(cast(size_t)dltotal, cast(size_t)dlnow, 664 cast(size_t)ultotal, cast(size_t)ulnow); 665 }