From: Galen Charlton Date: Wed, 7 Nov 2018 17:03:17 +0000 (-0500) Subject: LP#1729610: return new OpenSRF status if backlog queue fills up X-Git-Tag: osrf_rel_3_1_0-beta~4 X-Git-Url: http://git.equinoxoli.org/?p=opensrf-equinox.git;a=commitdiff_plain;h=efa9b713d5341458a3afaa26d1cf9e750fa78654 LP#1729610: return new OpenSRF status if backlog queue fills up This patch teaches Perl services how to return a new OpenSRF status, OSRF_STATUS_SERVICEUNAVAILABLE (code 503) if the backlog queue for a service gets full. To test ------- [1] Set a low max_backlog_queue for opensrf.sloooow and a low max_children. [2] Arrange for srfsh to fire off a bunch of opensrf.sloooow.wait requests. [3] Verify that requests that come in after the backlog queue fills up immediately get 503 exceptions. Signed-off-by: Galen Charlton Signed-off-by: Bill Erickson Signed-off-by: Mike Rylander --- diff --git a/include/opensrf/osrf_message.h b/include/opensrf/osrf_message.h index 1785adf..874c310 100644 --- a/include/opensrf/osrf_message.h +++ b/include/opensrf/osrf_message.h @@ -53,6 +53,7 @@ extern "C" { #define OSRF_STATUS_INTERNALSERVERERROR 500 #define OSRF_STATUS_NOTIMPLEMENTED 501 +#define OSRF_STATUS_SERVICEUNAVAILABLE 503 #define OSRF_STATUS_VERSIONNOTSUPPORTED 505 diff --git a/src/javascript/opensrf.js b/src/javascript/opensrf.js index e18efa5..03c79a9 100644 --- a/src/javascript/opensrf.js +++ b/src/javascript/opensrf.js @@ -58,6 +58,7 @@ var OSRF_STATUS_TIMEOUT = 408; var OSRF_STATUS_EXPFAILED = 417; var OSRF_STATUS_INTERNALSERVERERROR = 500; var OSRF_STATUS_NOTIMPLEMENTED = 501; +var OSRF_STATUS_SERVICEUNAVAILABLE = 503; var OSRF_STATUS_VERSIONNOTSUPPORTED = 505; // TODO: get path from ./configure prefix diff --git a/src/perl/lib/OpenSRF/DomainObject/oilsResponse.pm b/src/perl/lib/OpenSRF/DomainObject/oilsResponse.pm index 874a4ae..80fc7d2 100644 --- a/src/perl/lib/OpenSRF/DomainObject/oilsResponse.pm +++ b/src/perl/lib/OpenSRF/DomainObject/oilsResponse.pm @@ -12,6 +12,7 @@ BEGIN { STATUS_BADREQUEST STATUS_UNAUTHORIZED STATUS_FORBIDDEN STATUS_NOTFOUND STATUS_NOTALLOWED STATUS_TIMEOUT STATUS_INTERNALSERVERERROR STATUS_NOTIMPLEMENTED + STATUS_SERVICEUNAVAILABLE STATUS_VERSIONNOTSUPPORTED STATUS_REDIRECTED STATUS_EXPFAILED STATUS_COMPLETE STATUS_PARTIAL STATUS_NOCONTENT/; @@ -21,6 +22,7 @@ BEGIN { STATUS_BADREQUEST STATUS_UNAUTHORIZED STATUS_FORBIDDEN STATUS_NOTFOUND STATUS_NOTALLOWED STATUS_TIMEOUT STATUS_INTERNALSERVERERROR STATUS_NOTIMPLEMENTED + STATUS_SERVICEUNAVAILABLE STATUS_VERSIONNOTSUPPORTED STATUS_REDIRECTED STATUS_EXPFAILED STATUS_COMPLETE STATUS_PARTIAL STATUS_NOCONTENT/ ], @@ -72,6 +74,7 @@ sub STATUS_EXPFAILED { return 417 } sub STATUS_INTERNALSERVERERROR { return 500 } sub STATUS_NOTIMPLEMENTED { return 501 } +sub STATUS_SERVICEUNAVAILABLE { return 503 } sub STATUS_VERSIONNOTSUPPORTED { return 505 } my $log = 'OpenSRF::Utils::Logger'; diff --git a/src/perl/lib/OpenSRF/Server.pm b/src/perl/lib/OpenSRF/Server.pm index 0d31fcf..49ede59 100644 --- a/src/perl/lib/OpenSRF/Server.pm +++ b/src/perl/lib/OpenSRF/Server.pm @@ -21,6 +21,7 @@ use OpenSRF::Utils::Config; use OpenSRF::Transport::PeerHandle; use OpenSRF::Utils::SettingsClient; use OpenSRF::Utils::Logger qw($logger); +use OpenSRF::DomainObject::oilsResponse qw/:status/; use OpenSRF::Transport::SlimJabber::Client; use Encode; use POSIX qw/:sys_wait_h :errno_h/; @@ -227,8 +228,47 @@ sub run { } } } else { - # We'll just have to wait - $self->check_status(1); # block until child is available + + if (!$from_network) { + # The queue is full, and we just requeued a message. We'll + # now see if there is a request available from the network; + # if so, we'll see if a child is available again or else + # drop it + $msg = $self->{osrf_handle}->process($wait_time); + if ($msg) { + $self->check_status(); + if (@{$self->{idle_list}}) { + # child now available, so we'll go ahead and queue it + $chatty and $logger->debug("server: queuing new message after a re-queue with a full queue"); + push @max_children_msg_queue, $msg; + } else { + # ok, need to drop this one + my $resp = OpenSRF::DomainObject::oilsMessage->new(); + $resp->type('STATUS'); + $resp->payload( + OpenSRF::DomainObject::oilsMethodException->new( + status => "Service unavailable: no available children and backlog queue at limit", + statusCode => STATUS_SERVICEUNAVAILABLE + ) + ); + $resp->threadTrace(1); + + $logger->set_osrf_xid($msg->osrf_xid); + $self->{osrf_handle}->send( + to => $msg->from, + osrf_xid => $msg->osrf_xid, # Note that this is ignored, which + # is why we called $logger->set_osrf_xid above. + # We probably don't want that to be necessary + # if osrf_xid is explicitly set here, but that'll + # be a FIXME for later + thread => $msg->thread, + body => OpenSRF::Utils::JSON->perl2JSON([ $resp ]) + ); + $logger->warn("Backlog queue full for $self->{service}; forced to drop message " . + $msg->thread . " from " . $msg->from); + } + } + } } } diff --git a/src/python/osrf/const.py b/src/python/osrf/const.py index d888b75..297b1b3 100644 --- a/src/python/osrf/const.py +++ b/src/python/osrf/const.py @@ -63,6 +63,7 @@ OSRF_STATUS_TIMEOUT = 408 OSRF_STATUS_EXPFAILED = 417 OSRF_STATUS_INTERNALSERVERERROR = 500 OSRF_STATUS_NOTIMPLEMENTED = 501 +OSRF_STATUS_SERVICEUNAVAILABLE = 503 OSRF_STATUS_VERSIONNOTSUPPORTED = 505