request - help with streaming flash video through perlbal

[John Berthels]

On 22/11/2007, EJH Pollard <ejhpollard at gmail.com> wrote:
> I've been reading with interest the posts on the mogilefs mailing list
> from April 2007 regarding FLV (flash video) streaming through perlbal.
>  I'm asking for your help because I'm not 100% sure how to modify the
> patch at http://lists.danga.com/pipermail/mogilefs/2007-April/000862.html
> to enable some slightly different behaviour (for a free online archive
> of the UK parliament's video coverage).

Hi,

We've recently made the following local update to perlbal, heavily
based on the patch which went to the list, which adds support for two
more x-reproxy- headers back from the application:

x-reproxy-prefix - a hex encoded prefix to be prepended to the
reproxied response (a slightly generalised alternative to the original
patch's approach of hardcoding the FLV header in perlbal)

x-reproxy-additional-hdrs, a | separated list of additional headers to
add to the reproxied request (we need this since the original app
request isn't actually a partial GET, so we pass back an appropriate
Range: header to be included in the reproxied request).

(The patch also causes perlbal to advertise a new
x-reproxy-capability, 'reproxy-prefix', treating the header as a
comma-separated list).

Does anyone have any objections to my committing this?

regards,

jb

should apply cleanly to current perlbal with 'patch -p1' in top-level dir.

diff --git a/lib/Perlbal/BackendHTTP.pm b/lib/Perlbal/BackendHTTP.pm
index b6e774e..84f04ba 100644
--- a/lib/Perlbal/BackendHTTP.pm
+++ b/lib/Perlbal/BackendHTTP.pm
@@ -257,7 +257,7 @@ sub assign_client {
     $hds->header("Connection", $persist ? "keep-alive" : "close");

     if ($svc->{enable_reproxy}) {
-        $hds->header("X-Proxy-Capabilities", "reproxy-file");
+        $hds->header("X-Proxy-Capabilities", "reproxy-file, reproxy-prefix");
     }

     # decide whether we trust the upstream or not, to give us useful
@@ -453,6 +453,13 @@ sub handle_response { # : void

     my $reproxy_cache_for = $hd->header('X-REPROXY-CACHE-FOR') || 0;

+    if ($hd->header('X-REPROXY-ADDITIONAL-HDRS')) {
+        foreach my $line (split (/\|/,
$hd->header('X-REPROXY-ADDITIONAL-HDRS'))) {
+            my ($hdr, $val) = $line =~ /^(\S*):(.*$)/;
+            $client->{reproxy_additional_hdrs}->{$hdr} = $val;
+        }
+    }
+
     # special cases:  reproxying and retrying after server errors:
     if ((my $rep = $hd->header('X-REPROXY-FILE')) && $self->may_reproxy) {
         # make the client begin the async IO while we move on
@@ -493,12 +500,24 @@ sub handle_response { # : void
     # if we had an alternate primary response header, make sure
     # we send the real content-length (from the reproxied URL)
     # and not the one the first server gave us
+    my $prefix;
     if ($client->{primary_res_hdrs}) {
-        $thd->header('Content-Length', $hd->header('Content-Length'));
+        # If we are adding a prefix to the reproxy, adjust content-length
+        $prefix = pack('H*',
$client->{primary_res_hdrs}->header('X-REPROXY-PREFIX'))
+            if $client->{primary_res_hdrs}->header('X-REPROXY-PREFIX');
+        if (defined $prefix) {
+            $thd->header('Content-Length',
$hd->header('Content-Length') + length($prefix));
+        }
+        else {
+            $thd->header('Content-Length', $hd->header('Content-Length'));
+        }
+
         $thd->header('X-REPROXY-FILE', undef);
         $thd->header('X-REPROXY-URL', undef);
         $thd->header('X-REPROXY-EXPECTED-SIZE', undef);
         $thd->header('X-REPROXY-CACHE-FOR', undef);
+        $thd->header('X-REPROXY-PREFIX', undef);
+        $thd->header('X-REPROXY-ADDITIONAL-HDRS', undef);

         # also update the response code, in case of 206 partial content
         my $rescode = $hd->response_code;
@@ -508,6 +527,7 @@ sub handle_response { # : void

     print "  writing response headers to client\n" if Perlbal::DEBUG >= 3;
     $client->write($thd->to_string_ref);
+    $client->write($prefix) if $prefix;

     print("  content_length=", (defined $self->{content_length} ?
$self->{content_length} : "(undef)"),
           "  remain=",         (defined
$self->{content_length_remain} ? $self->{content_length_remain} :
"(undef)"), "\n")
diff --git a/lib/Perlbal/ClientProxy.pm b/lib/Perlbal/ClientProxy.pm
index 08d769f..7ff5f87 100644
--- a/lib/Perlbal/ClientProxy.pm
+++ b/lib/Perlbal/ClientProxy.pm
@@ -47,6 +47,7 @@ use fields (

             # error-retrying stuff
             'retry_count',         # number of times we've retried
this request so far after getting 500 errors
+            'reproxy_additional_hdrs',         # Headers we should
add to the reproxied request
             );

 use constant READ_SIZE         => 131072;    # 128k, ~common TCP window size?
@@ -112,6 +113,8 @@ sub init {
     $self->{currently_reproxying} = undef;

     $self->{retry_count} = 0;
+
+    $self->{reproxy_additional_hdrs} = undef;
 }

 # given a service name, re-request (GET/HEAD only) to that service, even though
@@ -234,6 +237,9 @@ sub use_reproxy_backend {
     if (my $host = $self->{req_headers}->header("Host")) {
         $extra_hdr .= "Host: $host\r\n";
     }
+    foreach my $hdr (keys %{$self->{reproxy_additional_hdrs}}) {
+        $extra_hdr .= "$hdr: $self->{reproxy_additional_hdrs}->{$hdr}\r\n";
+    }

     my $req_method = $self->{req_headers}->request_method eq 'HEAD' ?
'HEAD' : 'GET';
     my $headers = "$req_method $datref->[2] HTTP/1.0\r\nConnection:
keep-alive\r\n${extra_hdr}\r\n";
@@ -477,6 +483,7 @@ sub http_response_sent {
     $self->{upload_session} = undef;
     $self->{chunked_upload_state} = undef;
     $self->{request_body_length} = undef;
+    $self->{reproxy_additional_hdrs} = undef;
     return 1;
 }