1 module dast.net.curlapi;
2 
3 import dast.net.dlfcn;
4 import etc.c.curl : CurlGlobal;
5 import std.exception : enforce;
6 
7 /++
8 	Exception thrown on errors in std.net.curl functions.
9 +/
10 class CurlException : Exception {
11 	/++
12 		Params:
13 			msg  = The message for the exception.
14 			file = The file where the exception occurred.
15 			line = The line number where the exception occurred.
16 			next = The previous exception in the chain of exceptions, if any.
17 	+/
18 	@safe pure nothrow
19 	this(string msg,
20 		string file = __FILE__,
21 		size_t line = __LINE__,
22 		Throwable next = null) {
23 		super(msg, file, line, next);
24 	}
25 }
26 
27 /++
28 	Exception thrown on timeout errors in std.net.curl functions.
29 +/
30 class CurlTimeoutException : CurlException {
31 	/++
32 		Params:
33 			msg  = The message for the exception.
34 			file = The file where the exception occurred.
35 			line = The line number where the exception occurred.
36 			next = The previous exception in the chain of exceptions, if any.
37 	+/
38 	@safe pure nothrow
39 	this(string msg,
40 		string file = __FILE__,
41 		size_t line = __LINE__,
42 		Throwable next = null) {
43 		super(msg, file, line, next);
44 	}
45 }
46 
47 enum CurlInf : CurlInfo {
48 	contentlengthdownload = cast(CurlInfo)6_291_471,
49 	contentlengthupload = cast(CurlInfo)6_291_472,
50 }
51 
52 struct CurlAPI {
53 	import etc.c.curl : curl_version_info_data,
54 		CURL, CURLcode, CURLINFO, CURLoption, CURLversion, curl_slist;
55 
56 extern (C) nothrow @nogc:
57 	import core.stdc.config : c_long;
58 
59 	CURLcode function(c_long flags) global_init;
60 	void function() global_cleanup;
61 	curl_version_info_data* function(CURLversion) version_info;
62 	CURL* function() easy_init;
63 	CURLcode function(CURL* curl, CURLoption option, ...) easy_setopt;
64 	CURLcode function(CURL* curl) easy_perform;
65 	CURLcode function(CURL* curl, CURLINFO info, ...) easy_getinfo;
66 	CURL* function(CURL* curl) easy_duphandle;
67 	char* function(CURLcode) easy_strerror;
68 	CURLcode function(CURL* handle, int bitmask) easy_pause;
69 	void function(CURL* curl) easy_cleanup;
70 	curl_slist* function(curl_slist*, char*) slist_append;
71 	void function(curl_slist*) slist_free_all;
72 	void function(void*) free;
73 	char* function(CURL* curl, scope const(char)* s, int length) easy_escape;
74 	char* function(CURL* curl, scope const(char)* s, int length, int* outlength) easy_unescape;
75 
76 	CURL* function() multi_init;
77 	CURLMcode function(void* multi_handle, void* extra_fds, uint extra_nfds, int timeout_ms, int* ret) multi_wait;
78 	CURLMcode function(void* multi_handle, CURLMoption option, ...) multi_setopt;
79 	CURLMcode function(void* multi_handle, CURLMcode* running_handles) multi_perform;
80 	CURLMsg* function(void* multi_handle, int* msg_left) multi_info_read;
81 	CURLMcode function(void* multi_handle, CURL* easy_handle) multi_add_handle;
82 	CURLMcode function(void* multi_handle, CURL* easy_handle) multi_remove_handle;
83 	void function(void* multi_handle) multi_cleanup;
84 }
85 
86 private __gshared {
87 	CurlAPI _api;
88 	void* _handle;
89 }
90 
91 @property ref curlAPI() {
92 	import std.concurrency : initOnce;
93 
94 	initOnce!_handle(loadAPI());
95 	return _api;
96 }
97 
98 // TODO: use AliasSeq
99 version (LibcurlPath) {
100 	import std.string : strip;
101 
102 	private immutable names = [strip(import("LibcurlPathFile"))];
103 } else version (OSX)
104 	private immutable names = ["libcurl.4.dylib"];
105 else version (Posix) {
106 	private immutable names = [
107 		"libcurl.so", "libcurl.so.4",
108 		"libcurl-gnutls.so.4", "libcurl-nss.so.4", "libcurl.so.3"
109 	];
110 } else version (Windows)
111 	private immutable names = ["libcurl.dll", "curl.dll"];
112 
113 void* loadAPI() {
114 	import core.stdc.stdlib : atexit;
115 	import std.format : format;
116 
117 	void* handle;
118 	foreach (name; names) {
119 		handle = dlopen(name.ptr, RTLD_LAZY);
120 		if (handle !is null)
121 			break;
122 	}
123 
124 	enforce!CurlException(handle !is null, "Failed to load curl, tried %(%s, %).".format(names));
125 
126 	foreach (i, ref f; _api.tupleof) {
127 		enum name = __traits(identifier, _api.tupleof[i]);
128 		auto p = enforce!CurlException(dlsym(handle, "curl_" ~ name),
129 			"Couldn't load curl_" ~ name ~ " from libcurl.");
130 		f = cast(typeof(f))p;
131 	}
132 
133 	enforce!CurlException(!_api.global_init(CurlGlobal.all),
134 		"Failed to initialize libcurl");
135 
136 	atexit(&cleanup);
137 
138 	return handle;
139 }
140 
141 extern (C) void cleanup() {
142 	if (_handle is null)
143 		return;
144 	_api.global_cleanup();
145 
146 	dlclose(_handle);
147 	_api = CurlAPI.init;
148 	_handle = null;
149 }
150 
151 /**
152 	Wrapper to provide a better interface to libcurl than using the plain C API.
153 
154 	Warning: This struct uses interior pointers for callbacks. Only allocate it
155 	on the stack if you never move or copy it. This also means passing by reference
156 	when passing Curl to other functions. Otherwise always allocate on
157 	the heap.
158 */
159 struct Curl {
160 	import etc.c.curl;
161 
162 	alias OutData = void[];
163 	alias InData = ubyte[];
164 
165 	private {
166 		alias curl = curlAPI;
167 
168 		// A handle should not be used by two threads simultaneously
169 		CURL* ch;
170 		uint _rc;
171 
172 		// May also return `CURL_READFUNC_ABORT` or `CURL_READFUNC_PAUSE`
173 		size_t delegate(OutData) _onSend;
174 		size_t delegate(InData) _onReceive;
175 		void delegate(in char[]) _onReceiveHeader;
176 		CurlSeek delegate(long, CurlSeekPos) _onSeek;
177 		int delegate(curl_socket_t, CurlSockType) _onSocketOption;
178 		int delegate(size_t dltotal, size_t dlnow,
179 			size_t ultotal, size_t ulnow) _onProgress;
180 	}
181 
182 	//alias requestPause = CurlReadFunc.pause;
183 	//alias requestAbort = CurlReadFunc.abort;
184 
185 	this(CURL* h) {
186 		if (h)
187 			ch = h;
188 		else {
189 			ch = curl.easy_init();
190 			enforce!CurlException(ch, "Curl instance couldn't be initialized");
191 			set(CurlOption.nosignal, 1);
192 		}
193 		_rc = 1;
194 	}
195 
196 	@disable this();
197 
198 	this(this) {
199 		++_rc;
200 	}
201 
202 	~this() {
203 		if (ch)
204 			close();
205 	}
206 
207 	@property handle() => ch;
208 
209 	///
210 	@property bool stopped() const => ch is null;
211 
212 	/**
213 		Duplicate this handle.
214 
215 		The new handle will have all options set as the one it was duplicated
216 		from. An exception to this is that all options that cannot be shared
217 		across threads are reset thereby making it safe to use the duplicate
218 		in a new thread.
219 	*/
220 	Curl dup() {
221 		import std.meta : AliasSeq;
222 
223 		Curl copy = Curl(curl.easy_duphandle(ch));
224 
225 		with (CurlOption) {
226 			foreach (opt; AliasSeq!(file, writefunction, writeheader,
227 					headerfunction, infile, readfunction, ioctldata, ioctlfunction,
228 					seekdata, seekfunction, sockoptdata, sockoptfunction,
229 					opensocketdata, opensocketfunction, progressdata,
230 					progressfunction, debugdata, debugfunction, interleavedata,
231 					interleavefunction, chunk_data, chunk_bgn_function,
232 					chunk_end_function, fnmatch_data, fnmatch_function, cookiejar, postfields))
233 				copy.clear(opt);
234 
235 			// The options are only supported by libcurl when it has been built
236 			// against certain versions of OpenSSL - if your libcurl uses an old
237 			// OpenSSL, or uses an entirely different SSL engine, attempting to
238 			// clear these normally will raise an exception
239 			copy.tryClear(ssl_ctx_function);
240 			copy.tryClear(ssh_keydata);
241 		}
242 
243 		// Enable for curl version > 7.21.7
244 		static if (LIBCURL_VERSION_MAJOR >= 7 &&
245 			LIBCURL_VERSION_MINOR >= 21 &&
246 			LIBCURL_VERSION_PATCH >= 7) {
247 			copy.clear(CurlOption.closesocketdata);
248 			copy.clear(CurlOption.closesocketfunction);
249 		}
250 
251 		copy.set(CurlOption.nosignal, 1);
252 
253 		// copy.clear(CurlOption.ssl_ctx_data); Let ssl function be shared
254 		// copy.clear(CurlOption.ssh_keyfunction); Let key function be shared
255 
256 		/*
257 			Allow sharing of conv functions
258 			copy.clear(CurlOption.conv_to_network_function);
259 			copy.clear(CurlOption.conv_from_network_function);
260 			copy.clear(CurlOption.conv_from_utf8_function);
261 		*/
262 
263 		return copy;
264 	}
265 
266 	/**
267 		Stop and invalidate this curl instance.
268 		Warning: Do not call this from inside a callback handler e.g. `onReceive`.
269 	*/
270 	void close() {
271 		throwOnStopped();
272 		if (--_rc == 0)
273 			curl.easy_cleanup(ch);
274 		ch = null;
275 	}
276 
277 	/**
278 		Pausing and continuing transfers.
279 	*/
280 	void pause(bool sendingPaused, bool receivingPaused) {
281 		throwOnStopped();
282 		check(curl.easy_pause(ch,
283 				(sendingPaused ? CurlPause.send_cont
284 				: CurlPause.send) |
285 				(receivingPaused ? CurlPause.recv_cont : CurlPause.recv)));
286 	}
287 
288 	/**
289 		Set a string curl option.
290 		Params:
291 		option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation
292 		value = The string
293 	*/
294 	void set(CurlOption option, const(char)[] value) {
295 		import std.internal.cstring : tempCString;
296 
297 		throwOnStopped();
298 		check(curl.easy_setopt(ch, option, value.tempCString().buffPtr));
299 	}
300 
301 	/**
302 		Set a long curl option.
303 		Params:
304 		option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation
305 		value = The long
306 	*/
307 	void set(CurlOption option, long value) {
308 		throwOnStopped();
309 		check(curl.easy_setopt(ch, option, value));
310 	}
311 
312 	/**
313 		Set a void* curl option.
314 		Params:
315 		option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation
316 		value = The pointer
317 	*/
318 	void set(CurlOption option, void* value) {
319 		throwOnStopped();
320 		check(curl.easy_setopt(ch, option, value));
321 	}
322 
323 	/**
324 		Clear a pointer option.
325 		Params:
326 		option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation
327 	*/
328 	void clear(CurlOption option) {
329 		check(tryClear(option));
330 	}
331 
332 	/**
333 		Clear a pointer option. Does not raise an exception if the underlying
334 		libcurl does not support the option. Use sparingly.
335 		Params:
336 		option = A $(REF CurlOption, etc,c,curl) as found in the curl documentation
337 	*/
338 	CURLcode tryClear(CurlOption option) {
339 		throwOnStopped();
340 		return curl.easy_setopt(ch, option, null);
341 	}
342 
343 	/**
344 		perform the curl request by doing the HTTP,FTP etc. as it has
345 		been setup beforehand.
346 
347 		Params:
348 		throwOnError = whether to throw an exception or return a CURLcode on error
349 	*/
350 	CURLcode perform(bool throwOnError = true) {
351 		throwOnStopped();
352 		const code = curl.easy_perform(ch);
353 		if (throwOnError)
354 			check(code);
355 		return code;
356 	}
357 
358 	CURLcode tryGet(CurlInfo info, ref long val)
359 		=> curl.easy_getinfo(ch, info, &val);
360 
361 	long get(CurlInfo info) {
362 		long val;
363 		return curl.easy_getinfo(ch, info, &val) ? -1 : val;
364 	}
365 
366 	/**
367 	  * The event handler that receives incoming data.
368 	  *
369 	  * Params:
370 	  * callback = the callback that receives the `ubyte[]` data.
371 	  * Be sure to copy the incoming data and not store
372 	  * a slice.
373 	  *
374 	  * Returns:
375 	  * The callback returns the incoming bytes read. If not the entire array is
376 	  * the request will abort.
377 	  * The special value HTTP.pauseRequest can be returned in order to pause the
378 	  * current request.
379 	  *
380 	  * Example:
381 	  * ---
382 	  * import std.net.curl, std.stdio, std.conv;
383 	  * Curl curl;
384 	  * curl.initialize();
385 	  * curl.set(CurlOption.url, "http://dlang.org");
386 	  * curl.onReceive = (ubyte[] data) { writeln("Got data", to!(const(char)[])(data)); return data.length;};
387 	  * curl.perform();
388 	  * ---
389 	  */
390 	@property void onReceive(typeof(_onReceive) callback) {
391 		_onReceive = (InData id) {
392 			throwOnStopped("Receive callback called on cleaned up Curl instance");
393 			return callback(id);
394 		};
395 		set(CurlOption.file, &this);
396 		set(CurlOption.writefunction, &_receive);
397 	}
398 
399 	/**
400 	  * The event handler that receives incoming headers for protocols
401 	  * that uses headers.
402 	  *
403 	  * Params:
404 	  * callback = the callback that receives the header string.
405 	  * Make sure the callback copies the incoming params if
406 	  * it needs to store it because they are references into
407 	  * the backend and may very likely change.
408 	  *
409 	  * Example:
410 	  * ---
411 	  * import std.net.curl, std.stdio;
412 	  * Curl curl;
413 	  * curl.initialize();
414 	  * curl.set(CurlOption.url, "http://dlang.org");
415 	  * curl.onReceiveHeader = (in char[] header) { writeln(header); };
416 	  * curl.perform();
417 	  * ---
418 	  */
419 	@property void onReceiveHeader(void delegate(in char[]) callback) {
420 		_onReceiveHeader = (in char[] od) {
421 			throwOnStopped("Receive header callback called on " ~
422 					"cleaned up Curl instance");
423 			callback(od);
424 		};
425 		set(CurlOption.writeheader, &this);
426 		set(CurlOption.headerfunction, &_receiveHeader);
427 	}
428 
429 	/**
430 	  * The event handler that gets called when data is needed for sending.
431 	  *
432 	  * Params:
433 	  * callback = the callback that has a `void[]` buffer to be filled
434 	  *
435 	  * Returns:
436 	  * The callback returns the number of elements in the buffer that have been
437 	  * filled and are ready to send.
438 	  * The special value `Curl.abortRequest` can be returned in
439 	  * order to abort the current request.
440 	  * The special value `Curl.pauseRequest` can be returned in order to
441 	  * pause the current request.
442 	  *
443 	  * Example:
444 	  * ---
445 	  * import std.net.curl;
446 	  * Curl curl;
447 	  * curl.initialize();
448 	  * curl.set(CurlOption.url, "http://dlang.org");
449 	  *
450 	  * string msg = "Hello world";
451 	  * curl.onSend = (void[] data)
452 	  * {
453 	  *     auto m = cast(void[]) msg;
454 	  *     size_t length = m.length > data.length ? data.length : m.length;
455 	  *     if (length == 0) return 0;
456 	  *     data[0 .. length] = m[0 .. length];
457 	  *     msg = msg[length..$];
458 	  *     return length;
459 	  * };
460 	  * curl.perform();
461 	  * ---
462 	  */
463 	@property void onSend(typeof(_onSend) callback) {
464 		_onSend = (OutData od) {
465 			throwOnStopped("Send callback called on cleaned up Curl instance");
466 			return callback(od);
467 		};
468 		set(CurlOption.infile, &this);
469 		set(CurlOption.readfunction, &_send);
470 	}
471 
472 	/**
473 	  * The event handler that gets called when the curl backend needs to seek
474 	  * the data to be sent.
475 	  *
476 	  * Params:
477 	  * callback = the callback that receives a seek offset and a seek position
478 	  *            $(REF CurlSeekPos, etc,c,curl)
479 	  *
480 	  * Returns:
481 	  * The callback returns the success state of the seeking
482 	  * $(REF CurlSeek, etc,c,curl)
483 	  *
484 	  * Example:
485 	  * ---
486 	  * import std.net.curl;
487 	  * Curl curl;
488 	  * curl.initialize();
489 	  * curl.set(CurlOption.url, "http://dlang.org");
490 	  * curl.onSeek = (long p, CurlSeekPos sp)
491 	  * {
492 	  *     return CurlSeek.cantseek;
493 	  * };
494 	  * curl.perform();
495 	  * ---
496 	  */
497 	@property void onSeek(typeof(_onSeek) callback) {
498 		_onSeek = (long ofs, CurlSeekPos sp) {
499 			throwOnStopped("Seek callback called on cleaned up Curl instance");
500 			return callback(ofs, sp);
501 		};
502 		set(CurlOption.seekdata, &this);
503 		set(CurlOption.seekfunction, &_seek);
504 	}
505 
506 	/**
507 	  * The event handler that gets called when the net socket has been created
508 	  * but a `connect()` call has not yet been done. This makes it possible to set
509 	  * misc. socket options.
510 	  *
511 	  * Params:
512 	  * callback = the callback that receives the socket and socket type
513 	  * $(REF CurlSockType, etc,c,curl)
514 	  *
515 	  * Returns:
516 	  * Return 0 from the callback to signal success, return 1 to signal error
517 	  * and make curl close the socket
518 	  *
519 	  * Example:
520 	  * ---
521 	  * import std.net.curl;
522 	  * Curl curl;
523 	  * curl.initialize();
524 	  * curl.set(CurlOption.url, "http://dlang.org");
525 	  * curl.onSocketOption = delegate int(curl_socket_t s, CurlSockType t) { /+ do stuff +/ };
526 	  * curl.perform();
527 	  * ---
528 	  */
529 	@property void onSocketOption(typeof(_onSocketOption) callback) {
530 		_onSocketOption = (curl_socket_t sock, CurlSockType st) {
531 			throwOnStopped("Socket option callback called on " ~
532 					"cleaned up Curl instance");
533 			return callback(sock, st);
534 		};
535 		set(CurlOption.sockoptdata, &this);
536 		set(CurlOption.sockoptfunction, &_socketOption);
537 	}
538 
539 	/**
540 	  * The event handler that gets called to inform of upload/download progress.
541 	  *
542 	  * Params:
543 	  * callback = the callback that receives the (total bytes to download,
544 	  * currently downloaded bytes, total bytes to upload, currently uploaded
545 	  * bytes).
546 	  *
547 	  * Returns:
548 	  * Return 0 from the callback to signal success, return non-zero to abort
549 	  * transfer
550 	  *
551 	  * Example:
552 	  * ---
553 	  * import std.net.curl, std.stdio;
554 	  * Curl curl;
555 	  * curl.initialize();
556 	  * curl.set(CurlOption.url, "http://dlang.org");
557 	  * curl.onProgress = delegate int(size_t dltotal, size_t dlnow, size_t ultotal, size_t ulnow)
558 	  * {
559 	  *     writeln("Progress: downloaded bytes ", dlnow, " of ", dltotal);
560 	  *     writeln("Progress: uploaded bytes ", ulnow, " of ", ultotal);
561 	  *     return 0;
562 	  * };
563 	  * curl.perform();
564 	  * ---
565 	  */
566 	@property void onProgress(typeof(_onProgress) callback) {
567 		_onProgress = (size_t dlt, size_t dln, size_t ult, size_t uln) {
568 			throwOnStopped("Progress callback called on cleaned " ~
569 					"up Curl instance");
570 			return callback(dlt, dln, ult, uln);
571 		};
572 		set(CurlOption.noprogress, 0);
573 		set(CurlOption.progressdata, &this);
574 		set(CurlOption.progressfunction, &_progress);
575 	}
576 
577 private:
578 
579 	void check(CURLcode code) {
580 		enforce!CurlTimeoutException(code != CurlError.operation_timedout,
581 			errorString(code));
582 
583 		enforce!CurlException(code == CurlError.ok,
584 			errorString(code));
585 	}
586 
587 	string errorString(CURLcode code) {
588 		import std.string : fromStringz;
589 		import std.format : format;
590 
591 		const msgZ = curl.easy_strerror(code);
592 		// doing the following (instead of just using std.conv.to!string) avoids 1 allocation
593 		return "%s on handle %s".format(fromStringz(msgZ), ch);
594 	}
595 
596 	void throwOnStopped(string message = null) {
597 		enum def = "Curl instance called after being cleaned up";
598 		enforce!CurlException(!stopped,
599 			message == null ? def : message);
600 	}
601 }
602 
603 import etc.c.curl;
604 
605 private extern (C):
606 // Internal C callbacks to register with libcurl
607 size_t _receive(const char* str,
608 	size_t size, size_t nmemb, void* ptr) {
609 	auto b = cast(Curl*)ptr;
610 	if (b._onReceive != null)
611 		return b._onReceive(cast(Curl.InData)str[0 .. size * nmemb]);
612 	return size * nmemb;
613 }
614 
615 size_t _receiveHeader(const char* str,
616 	size_t size, size_t nmemb, void* ptr) {
617 	import std.string : chomp;
618 
619 	auto b = cast(Curl*)ptr;
620 	const s = str[0 .. size * nmemb].chomp();
621 	if (b._onReceiveHeader != null)
622 		b._onReceiveHeader(s);
623 
624 	return size * nmemb;
625 }
626 
627 size_t _send(char* str, size_t size, size_t nmemb, void* ptr) {
628 	Curl* b = cast(Curl*)ptr;
629 	auto a = cast(void[])str[0 .. size * nmemb];
630 	if (b._onSend == null)
631 		return 0;
632 	return b._onSend(a);
633 }
634 
635 int _seek(void* ptr, curl_off_t offset, int origin) {
636 	auto b = cast(Curl*)ptr;
637 	if (b._onSeek == null)
638 		return CurlSeek.cantseek;
639 
640 	// origin: CurlSeekPos.set/current/end
641 	// return: CurlSeek.ok/fail/cantseek
642 	return b._onSeek(cast(long)offset, cast(CurlSeekPos)origin);
643 }
644 
645 int _socketOption(void* ptr,
646 	curl_socket_t curlfd, curlsocktype purpose) {
647 	auto b = cast(Curl*)ptr;
648 	if (b._onSocketOption == null)
649 		return 0;
650 
651 	// return: 0 ok, 1 fail
652 	return b._onSocketOption(curlfd, cast(CurlSockType)purpose);
653 }
654 
655 int _progress(void* ptr,
656 	double dltotal, double dlnow,
657 	double ultotal, double ulnow) {
658 	auto b = cast(Curl*)ptr;
659 	if (b._onProgress == null)
660 		return 0;
661 
662 	// return: 0 ok, 1 fail
663 	return b._onProgress(cast(size_t)dltotal, cast(size_t)dlnow,
664 		cast(size_t)ultotal, cast(size_t)ulnow);
665 }