2 # ABSTRACT: An HTTP user agent programming interface unification layer
9 our $VERSION = '0.900'; # VERSION
11 use HTTP
::AnyUA
::Util
;
16 our $BACKEND_NAMESPACE;
18 our %REGISTERED_BACKENDS;
21 $BACKEND_NAMESPACE = __PACKAGE__
. '::Backend';
25 sub _debug_log
{ print STDERR
join(' ', @_), "\n" if $ENV{PERL_HTTP_ANYUA_DEBUG
} }
27 sub _croak
{ require Carp
; Carp
::croak
(@_) }
28 sub _usage
{ _croak
("Usage: @_\n") }
34 unshift @_, 'ua' if @_ % 2;
36 $args{ua
} or _usage
(q{HTTP::AnyUA->new(ua => $user_agent, %attr)});
39 my @attr = qw(ua backend response_is_future);
41 for my $attr (@attr) {
42 $self->{$attr} = $args{$attr} if defined $args{$attr};
47 $self->_debug_log('Created with user agent', $self->ua);
49 # call accessors to get the checks to run
51 $self->response_is_future($args{response_is_future
}) if defined $args{response_is_future
};
57 sub ua
{ shift-
>{ua
} or _croak
'User agent is required' }
60 sub response_is_future
{
65 $self->_debug_log('Set response_is_future to', $val ? 'ON' : 'OFF');
67 $self->_check_response_is_future($val);
68 $self->{response_is_future
} = $val;
70 $self->_module_loader->load('Future') if $self->{response_is_future
};
72 elsif (!defined $self->{response_is_future
} && $self->{backend
}) {
73 $self->{response_is_future
} = $self->backend->response_is_future;
75 $self->_module_loader->load('Future') if $self->{response_is_future
};
78 return $self->{response_is_future
} || '';
85 return $self->{backend
} if defined $self->{backend
};
87 $self->{backend
} = $self->_build_backend;
88 $self->_check_response_is_future($self->response_is_future);
90 return $self->{backend
};
95 my ($self, $method, $url, $args) = @_;
97 @_ == 3 || (@_ == 4 && ref $args eq 'HASH')
98 or _usage
(q{$any_ua->request($method, $url, \%options)});
100 my $resp = eval { $self->backend->request(uc($method) => $url, $args) };
102 return $self->_wrap_internal_exception($err);
105 return $self->_wrap_response($resp);
109 # adapted from HTTP/Tiny.pm
110 for my $sub_name (qw{get head put post delete}) {
111 my %swap = (SUBNAME
=> $sub_name, METHOD
=> uc($sub_name));
114 my ($self, $url, $args) = @_;
115 @_ == 2 || (@_ == 3 && ref $args eq 'HASH')
116 or _usage(q{$any_ua->{{SUBNAME}}($url, \
%options)});
117 return $self->request('{{METHOD}}', $url, $args);
120 $code =~ s/\{\{([A-Z_]+)\}\}/$swap{$1}/ge;
121 eval $code; ## no critic
125 # adapted from HTTP/Tiny.pm
127 my ($self, $url, $data, $args) = @_;
128 (@_ == 3 || @_ == 4 && ref $args eq 'HASH')
129 or _usage
(q{$any_ua->post_form($url, $formdata, \%options)});
132 while (my ($key, $value) = each %{$args->{headers
} || {}}) {
133 $headers->{lc $key} = $value;
135 delete $args->{headers
};
137 return $self->request(POST
=> $url, {
139 content
=> HTTP
::AnyUA
::Util
::www_form_urlencode
($data),
142 'content-type' => 'application/x-www-form-urlencoded',
148 # adapted from HTTP/Tiny.pm
150 my ($self, $url, $file, $args) = @_;
151 @_ == 3 || (@_ == 4 && ref $args eq 'HASH')
152 or _usage
(q{$any_ua->mirror($url, $filepath, \%options)});
154 if (exists $args->{headers
}) {
156 while (my ($key, $value) = each %{$args->{headers
} || {}}) {
157 $headers->{lc($key)} = $value;
159 $args->{headers
} = $headers;
162 if (-e
$file and my $mtime = (stat($file))[9]) {
163 $args->{headers
}{'if-modified-since'} ||= HTTP
::AnyUA
::Util
::http_date
($mtime);
165 my $tempfile = $file . int(rand(2**31));
167 # set up the response body to be written to the file
169 sysopen(my $fh, $tempfile, Fcntl
::O_CREAT
()|Fcntl
::O_EXCL
()|Fcntl
::O_WRONLY
())
170 or return $self->_wrap_internal_exception(qq
/Error: Could not create temporary file $tempfile for downloading: $!\n/);
172 $args->{data_callback
} = sub { print $fh $_[0] };
174 my $resp = $self->request(GET
=> $url, $args);
180 or return HTTP
::AnyUA
::Util
::internal_exception
(qq
/Error: Caught error closing temporary file $tempfile: $!\n/);
182 if ($resp->{success
}) {
183 rename($tempfile, $file)
184 or return HTTP
::AnyUA
::Util
::internal_exception
(qq
/Error replacing $file with $tempfile: $!\n/);
185 my $lm = $resp->{headers
}{'last-modified'};
186 if ($lm and my $mtime = HTTP
::AnyUA
::Util
::parse_http_date
($lm)) {
187 utime($mtime, $mtime, $file);
192 $resp->{success
} ||= $resp->{status
} eq '304';
197 if ($self->response_is_future) {
198 return $resp->followed_by(sub {
200 my @resp = $future->is_done ? $future->get : $future->failure;
201 my $resp = $finish->(@resp);
202 if ($resp->{success
}) {
203 return Future-
>done(@resp);
206 return Future-
>fail(@resp);
211 return $finish->($resp);
216 sub register_backend
{
217 my ($class, $ua_type, $backend_class) = @_;
218 @_ == 3 or _usage
(q{HTTP::AnyUA->register_backend($ua_type, $backend_package)});
220 if ($backend_class) {
221 $backend_class = "${BACKEND_NAMESPACE}::${backend_class}" unless $backend_class =~ s/^\+//;
222 $REGISTERED_BACKENDS{$ua_type} = $backend_class;
225 delete $REGISTERED_BACKENDS{$ua_type};
230 # turn a response into a Future if it needs to be
235 if ($self->response_is_future && !$self->backend->response_is_future) {
236 # wrap the response in a Future
237 if ($resp->{success
}) {
238 $self->_debug_log('Wrapped successful response in a Future');
239 $resp = Future-
>done($resp);
242 $self->_debug_log('Wrapped failed response in a Future');
243 $resp = Future-
>fail($resp);
250 sub _wrap_internal_exception
{ shift-
>_wrap_response(HTTP
::AnyUA
::Util
::internal_exception
(@_)) }
252 # get a module loader object
253 sub _module_loader
{ shift-
>{_module_loader
} ||= Module
::Loader-
>new }
255 # get a list of potential backends that may be able to handle the user agent
258 my $ua = shift || $self->ua or _croak
'User agent is required';
260 my $ua_type = Scalar
::Util
::blessed
($ua);
265 push @classes, $REGISTERED_BACKENDS{$ua_type} if $REGISTERED_BACKENDS{$ua_type};
267 push @classes, "${BACKEND_NAMESPACE}::${ua_type}";
270 # search for some backends to try
271 @BACKENDS = sort $self->_module_loader->find_modules($BACKEND_NAMESPACE);
272 $self->_debug_log('Found backends to try (' . join(', ', @BACKENDS) . ')');
275 for my $backend_type (@BACKENDS) {
276 my $plugin = $backend_type;
277 $plugin =~ s/^\Q${BACKEND_NAMESPACE}\E:://;
278 push @classes, $backend_type if $ua->isa($plugin);
282 push @classes, $REGISTERED_BACKENDS{$ua} if $REGISTERED_BACKENDS{$ua};
283 push @classes, "${BACKEND_NAMESPACE}::${ua}";
286 for my $class (@classes) {
287 if (eval { $self->_module_loader->load($class); 1 }) {
288 $self->_debug_log("Found usable backend (${class})");
289 return $class->new($self->ua);
292 $self->_debug_log($@);
296 _croak
'Cannot find a usable backend that supports the given user agent';
299 # make sure the response_is_future setting is compatible with the backend
300 sub _check_response_is_future
{
304 # make sure the user agent is not non-blocking
305 if (!$val && $self->{backend
} && $self->backend->response_is_future) {
306 _croak
'Cannot disable response_is_future with a non-blocking user agent';
320 HTTP::AnyUA - An HTTP user agent programming interface unification layer
328 my $any_ua = HTTP::AnyUA->new(ua => LWP::UserAgent->new);
329 # OR: my $any_ua = HTTP::AnyUA->new(ua => Furl->new);
330 # OR: my $any_ua = HTTP::AnyUA->new(ua => HTTP::Tiny->new);
333 my $response = $any_ua->get('http://www.example.com/');
335 print "$response->{status} $response->{reason}\n";
337 while (my ($k, $v) = each %{$response->{headers}}) {
338 for (ref $v eq 'ARRAY' ? @$v : $v) {
343 print $response->{content} if length $response->{content};
345 ### Non-blocking user agents cause Future objects to be returned:
347 my $any_ua = HTTP::AnyUA->new(ua => HTTP::Tiny->new, response_is_future => 1);
348 # OR: my $any_ua = HTTP::AnyUA->new(ua => 'AnyEvent::HTTP');
349 # OR: my $any_ua = HTTP::AnyUA->new(ua => Mojo::UserAgent->new);
352 my $future = $any_ua->get('http://www.example.com/');
354 $future->on_done(sub {
355 my $response = shift;
357 print "$response->{status} $response->{reason}\n";
359 while (my ($k, $v) = each %{$response->{headers}}) {
360 for (ref $v eq 'ARRAY' ? @$v : $v) {
365 print $response->{content} if length $response->{content};
368 $future->on_fail(sub { print STDERR "Oh no!!\n" });
372 This module provides a small wrapper for unifying the programming interfaces of several different
373 actual user agents (HTTP clients) under one B<familiar> interface.
375 Rather than providing yet another programming interface for you to learn, HTTP::AnyUA follows the
376 L<HTTP::Tiny> interface. This also means that you can plug in any supported HTTP client
377 (L<LWP::UserAgent>, L<Furl>, etc.) and use it as if it were L<HTTP::Tiny>.
379 There are a lot of great HTTP clients available for Perl, each with different goals, different
380 feature sets, and of course different programming interfaces! If you're an end user, you can just
381 pick one of these clients according to the needs of your project (or personal preference). But if
382 you're writing a module that needs to interface with a web server (like perhaps a RESTful API
383 wrapper) and you want your users to be able to use whatever HTTP client they want, HTTP::AnyUA can
384 help you support that!
386 It's a good idea to let the end user pick whatever HTTP client they want to use, because they're the
387 one who knows the requirements of their application or script. If you're writing an event-driven
388 application, you'll need to use a non-blocking user agent like L<Mojo::UserAgent>. If you're writing
389 a simple command-line script, you may decide that your priority is to minimize dependencies and so
390 may want to go with L<HTTP::Tiny>.
392 Unfortunately, many modules on CPAN are hardcoded to work with specific HTTP clients, leaving the
393 end user unable to use the HTTP client that would be best for them. Although the end user won't --
394 or at least doesn't need to -- use HTTP::AnyUA directly, they will benefit from client choice if
395 their third-party modules use HTTP::AnyUA or something like it.
397 The primary goal of HTTP::AnyUA is to make it easy for module developers to write HTTP code once
398 that can work with any HTTP client the end user may decide to plug in. A secondary goal is to make
399 it easy for anyone to add support for new or yet-unsupported user agents.
405 Get the user agent that was passed to L</new>.
407 =head2 response_is_future
409 Get and set whether or not responses are L<Future> objects.
413 Get the backend instance. You normally shouldn't need this.
419 $any_ua = HTTP::AnyUA->new(ua => $user_agent, %attr);
420 $any_ua = HTTP::AnyUA->new($user_agent, %attr);
422 Construct a new HTTP::AnyUA.
426 $response = $any_ua->request($method, $url);
427 $response = $any_ua->request($method, $url, \%options);
429 Make a L<request|/"The Request">, get a L<response|/"The Response">.
431 Compare to L<HTTP::Tiny/request>.
433 =head2 get, head, put, post, delete
435 $response = $any_ua->get($url);
436 $response = $any_ua->get($url, \%options);
437 $response = $any_ua->head($url);
438 $response = $any_ua->head($url, \%options);
441 Shortcuts for L</request> where the method is the method name rather than the first argument.
443 Compare to L<HTTP::Tiny/getE<verbar>headE<verbar>putE<verbar>postE<verbar>delete>.
447 $response = $any_ua->post_form($url, $formdata);
448 $response = $any_ua->post_form($url, $formdata, \%options);
450 Does a C<POST> request with the form data encoded and sets the C<Content-Type> header to
451 C<application/x-www-form-urlencoded>.
453 Compare to L<HTTP::Tiny/post_form>.
457 $response = $http->mirror($url, $filepath, \%options);
458 if ($response->{success}) {
459 print "$filepath is up to date\n";
462 Does a C<GET> request and saves the downloaded document to a file. If the file already exists, its
463 timestamp will be sent using the C<If-Modified-Since> request header (which you can override). If
464 the server responds with a C<304> (Not Modified) status, the C<success> field will be true; this is
465 usually only the case for C<2XX> statuses. If the server responds with a C<Last-Modified> header,
466 the file will be updated to have the same modification timestamp.
468 Compare to L<HTTP::Tiny/mirror>. This version differs slightly in that this returns internal
469 exception responses (for cases like being unable to write the file locally, etc.) rather than
470 actually throwing the exceptions. The reason for this is that exceptions as responses are easier to
471 deal with for non-blocking HTTP clients, and the fact that this method throws exceptions in
472 L<HTTP::Tiny> seems like an inconsistency in its interface.
474 =head2 register_backend
476 HTTP::AnyUA->register_backend($user_agent_package => $backend_package);
477 HTTP::AnyUA->register_backend('MyAgent' => 'MyBackend'); # HTTP::AnyUA::Backend::MyBackend
478 HTTP::AnyUA->register_backend('LWP::UserAgent' => '+SpecialBackend'); # SpecialBackend
480 Register a backend for a new user agent type or override a default backend. Backend packages are
481 relative to the C<HTTP::AnyUA::Backend::> namespace unless prefixed with a C<+>.
483 If you only need to set a backend as a one-off thing, you could also pass an instantiated backend to
486 =head1 SUPPORTED USER AGENTS
500 L<HTTP::AnyUA> - a little bit meta, but why not?
520 Any HTTP client that inherits from one of these in a well-behaved manner should also be supported.
522 Of course, there are many other HTTP clients on CPAN that HTTP::AnyUA doesn't yet support. I'm more
523 than happy to help add support for others, so send me a message if you know of an HTTP client that
524 needs support. See L<HTTP::AnyUA::Backend> for how to write support for a new HTTP client.
526 =head1 NON-BLOCKING USER AGENTS
528 HTTP::AnyUA tries to target the L<HTTP::Tiny> interface, which is a blocking interface. This means
529 that when you call L</request>, it is supposed to not return until either the response is received
530 or an error occurs. This doesn't jive well with non-blocking HTTP clients which expect the flow to
531 reenter an event loop so that the request can complete concurrently.
533 In order to reconcile this, a L<Future> will be returned instead of the normal hashref response if
534 the wrapped HTTP client is non-blocking (such as L<Mojo::UserAgent> or L<AnyEvent::HTTP>). This
535 L<Future> object may be used to set up callbacks that will be called when the request is completed.
536 You can call L</response_is_future> to know if the response is or will be a L<Future>.
538 This is typically okay for the end user; since they're the one who chose which HTTP client to use in
539 the first place, they should know whether they should expect a L<Future> or a direct response when
540 they make an HTTP request, but it does add some burden on you as a module writer because if you ever
541 need to examine the response, you may need to write code like this:
543 my $resp = $any_ua->get('http://www.perl.org/');
545 if ($any_ua->response_is_future) {
547 my $real_resp = shift;
548 handle_response($real_resp);
552 handle_response($resp); # response is the real response already
555 This actually isn't too annoying to deal with in practice, but you can avoid it if you like by
556 forcing the response to always be a L<Future>. Just set the L</response_is_future> attribute. Then
557 you don't need to do an if-else because the response will always be the same type:
559 $any_ua->response_is_future(1);
561 my $resp = $any_ua->get('http://www.perl.org/');
563 $resp->on_done(sub { # response is always a Future
564 my $real_resp = shift;
565 handle_response($real_resp);
568 Note that this doesn't make a blocking HTTP client magically non-blocking. The call to L</request>
569 will still block if the client is blocking, and your "done" callback will simply be fired
570 immediately. But this does let you write the same code in your module and have it work regardless of
571 whether the underlying HTTP client is blocking or non-blocking.
573 The default behavior is to return a direct hashref response if the HTTP client is blocking and
574 a L<Future> if the client is non-blocking. It's up to you to decide whether or not to set
575 C<response_is_future>, and you should also consider whether you want to expose the possibility of
576 either type of response or always returning L<Future> objects to the end user of your module. It
577 doesn't matter for users who choose non-blocking HTTP clients because they will be using L<Future>
578 objects either way, but users who know they are using a blocking HTTP client may appreciate not
579 having to deal with L<Future> objects at all.
581 =head1 FREQUENTLY ASKED QUESTIONS
583 =head2 How do I set up proxying, SSL, cookies, timeout, etc.?
585 HTTP::AnyUA provides a common interface for I<using> HTTP clients, not for instantiating or
586 configuring them. Proxying, SSL, and other custom settings can be configured directly through the
587 underlying HTTP client; see the documentation for your particular user agent to learn how to
588 configure these things.
590 L<AnyEvent::HTTP> is a bit of a special case because there is no instantiated object representing
591 the client. For this particular user agent, you can configure the backend to pass a default set of
592 options whenever it calls C<http_request>. See L<HTTP::AnyUA::Backend::AnyEvent::HTTP/options>:
594 $any_ua->backend->options({recurse => 5, timeout => 15});
596 If you are a module writer, you should probably receive a user agent from your end user and leave
597 this type of configuration up to them.
599 =head2 Why use HTTP::AnyUA instead of some other HTTP client?
601 Maybe you shouldn't. If you're an end user writing a script or application, you can just pick the
602 HTTP client that suits you best and use it. For example, if you're writing a L<Mojolicious> app,
603 you're not going wrong by using L<Mojo::UserAgent>; it's loaded with features and is well-integrated
604 with that particular environment.
606 As an end user, you I<could> wrap the HTTP client you pick in an HTTP::AnyUA object, but the only
607 reason to do this is if you prefer using the L<HTTP::Tiny> interface.
609 The real benefit of HTTP::AnyUA (or something like it) is if module writers use it to allow end
610 users of their modules to be able to plug in whatever HTTP client they want. For example, a module
611 that implements an API wrapper that has a hard dependency on L<LWP::UserAgent> or even L<HTTP::Tiny>
612 is essentially useless for non-blocking applications. If the same hypothetical module had been
613 written using HTTP::AnyUA then it would be useful in any scenario.
615 =head2 Why use the HTTP::Tiny interface?
617 The L<HTTP::Tiny> interface is simple but provides all the essential functionality needed for
618 a capable HTTP client and little more. That makes it easy to provide an implementation for, and it
619 also makes it straightforward for module authors to use.
621 Marrying the L<HTTP::Tiny> interface with L<Future> gives us these benefits for both blocking and
622 non-blocking modules and applications.
626 This section specifies a standard set of data structures that can be used to make a request and get
627 a response from a user agent. This is the specification HTTP::AnyUA uses for its programming
628 interface. It is heavily based on L<HTTP::Tiny>'s interface, and parts of this specification were
629 adapted or copied verbatim from that module's documentation. The intent is for this specification to
630 be written such that L<HTTP::Tiny> is already a compliant implementor of the specification (at least
631 as of the specification's publication date).
635 A request is a tuple of the form C<(Method, URL)> or C<(Method, URL, Options)>.
639 Method B<MUST> be a string representing the HTTP verb. This is commonly C<"GET">, C<"POST">,
640 C<"HEAD">, C<"DELETE">, etc.
644 URL B<MUST> be a string representing the remote resource to be acted upon. The URL B<MUST> have
645 unsafe characters escaped and international domain names encoded before being passed to the user
646 agent. A user agent B<MUST> generated a C<"Host"> header based on the URL in accordance with RFC
647 2616; a user agent B<MAY> throw an error if a C<"Host"> header is given with the L</headers>.
651 Options, if present, B<MUST> be a hash reference containing zero or more of the following keys with
652 appropriate values. A user agent B<MAY> support more options than are specified here.
656 The value for the C<headers> key B<MUST> be a hash reference containing zero or more HTTP header
657 names (as keys) and header values. The value for a header B<MUST> be either a string containing the
658 header value OR an array reference where each item is a string. If the value for a header is an
659 array reference, the user agent B<MUST> output the header multiple times with each value in the
662 User agents B<MAY> may add headers, but B<SHOULD NOT> replace user-specified headers unless
663 otherwise documented.
667 The value for the C<content> key B<MUST> be a string OR a code reference. If the value is a string,
668 its contents will be included with the request as the body. If the value is a code reference, the
669 referenced code will be called iteratively to produce the body of the request, and the code B<MUST>
670 return an empty string or undef value to indicate the end of the request body. If the value is
671 a code reference, a user agent B<SHOULD> use chunked transfer encoding if it supports it, otherwise
672 a user agent B<MAY> completely drain the code of content before sending the request.
676 The value for the C<data_callback> key B<MUST> be a code reference that will be called zero or more
677 times, once for each "chunk" of response body received. A user agent B<MAY> send the entire response
678 body in one call. The referenced code B<MUST> be given two arguments; the first is a string
679 containing a chunk of the response body, the second is an in-progress L<response|/The Response>.
683 A response B<MUST> be a hash reference containg some required keys and values. A response B<MAY>
684 contain some optional keys and values.
688 A response B<MUST> include a C<success> key, the value of which is a boolean indicating whether or
689 not the request is to be considered a success (true is a success). Unless otherwise documented,
690 a successful result means that the operation returned a 2XX status code.
694 A response B<MUST> include a C<url> key, the value of which is the URL that provided the response.
695 This is the URL used in the request unless there were redirections, in which case it is the last URL
696 queried in a rediretion chain.
700 A response B<MUST> include a C<status> key, the value of which is the HTTP status code of the
701 response. If an internal exception occurs (e.g. connection error), then the status code B<MUST> be
706 A response B<MUST> include a C<reason> key, the value of which is the response phrase returned by
707 the server OR "Internal Exception" if an internal exception occurred.
711 A response B<MAY> include a C<content> key, the value of which is the response body returned by the
712 server OR the text of the exception if an internal exception occurred. This field B<MUST> be missing
713 or empty if the server provided no response OR if the body was already provided via
718 A response B<SHOULD> include a C<headers> key, the value of which is a hash reference containing
719 zero or more HTTP header names (as keys) and header values. Keys B<MUST> be lowercased. The value
720 for a header B<MUST> be either a string containing the header value OR an array reference where each
721 item is the value of one of the repeated headers.
725 A response B<MAY> include a C<redirects> key, the value of which is an array reference of one or
726 more responses from redirections that occurred to fulfill the current request, in chronological
735 C<PERL_HTTP_ANYUA_DEBUG> - If 1, print some info useful for debugging to C<STDERR>.
741 Not all HTTP clients implement the same features or in the same ways. While the point of HTTP::AnyUA
742 is to hide those differences, you may notice some (hopefully) I<insignificant> differences when
743 plugging in different clients. For example, L<LWP::UserAgent> sets some headers on the response such
744 as C<client-date> and C<client-peer> that won't appear when using other clients. Little differences
745 like these probably aren't big deal. Other differences may be a bigger deal, depending on what's
746 important to you. For example, some clients (like L<HTTP::Tiny>) may do chunked transfer encoding in
747 situations where other clients won't (probably because they don't support it). It's not a goal of
748 this project to eliminate I<all> of the differences, but if you come across a difference that is
749 significant enough that you think you need to detect the user agent and write special logic, I would
750 like to learn about your use case.
754 These modules share similar goals or provide overlapping functionality:
778 Please report any bugs or feature requests on the bugtracker website
779 L<https://github.com/chazmcgarvey/HTTP-AnyUA/issues>
781 When submitting a bug or request, please include a test-file or a
782 patch to an existing test-file that illustrates the bug or desired
787 Charles McGarvey <chazmcgarvey@brokenzipper.com>
789 =head1 COPYRIGHT AND LICENSE
791 This software is copyright (c) 2017 by Charles McGarvey.
793 This is free software; you can redistribute it and/or modify it under
794 the same terms as the Perl 5 programming language system itself.