[PATCH 2/7] Move main backup routine into classes

Michael Hanselmann public at hansmi.ch
Wed Aug 13 21:15:57 UTC 2008


From: Michael Hanselmann <hansmi at hansmi-mac-zrh.local>

Encryption and status information have been left out in this patch.
They'll be reimplemented in consecutive patches.
---
 lib/Brackup/Backup.pm |  518 +++++++++++++++++++++++++------------------------
 1 files changed, 264 insertions(+), 254 deletions(-)

diff --git a/lib/Brackup/Backup.pm b/lib/Brackup/Backup.pm
index 7211439..dfc4818 100644
--- a/lib/Brackup/Backup.pm
+++ b/lib/Brackup/Backup.pm
@@ -17,10 +17,6 @@ sub new {
     $self->{verbose} = delete $opts{verbose};  # bool
     $self->{zenityprogress} = delete $opts{zenityprogress};  # bool
 
-    $self->{modecounts} = {}; # type -> mode(octal) -> count
-
-    $self->{saved_files} = [];   # list of Brackup::File objects backed up
-
     croak("Unknown options: " . join(', ', keys %opts)) if %opts;
 
     return $self;
@@ -29,251 +25,259 @@ sub new {
 # returns true (a Brackup::BackupStats object) on success, or dies with error
 sub backup {
     my ($self, $backup_file) = @_;
+    my $backup = Brackup::Backup::_Sync->new($self, $backup_file);
 
-    my $root   = $self->{root};
-    my $target = $self->{target};
+    $backup->run();
 
-    my $stats  = Brackup::BackupStats->new;
+    return $backup->stats;
+}
 
-    my $gpg_rcpt = $self->{root}->gpg_rcpt;
+sub backup_time {
+    my $self = shift;
+    return $self->{backup_time} ||= time();
+}
 
-    my $n_kb         = 0.0; # num:  kb of all files in root
-    my $n_files      = 0;   # int:  # of files in root
-    my $n_kb_done    = 0.0; # num:  kb of files already done with (uploaded or skipped)
 
-    # if we're pre-calculating the amount of data we'll
-    # actually need to upload, store it here.
-    my $n_kb_up      = 0.0;
-    my $n_kb_up_need = 0.0; # by default, not calculated/used.
+package Brackup::Backup::_Base;
+use strict;
+use warnings;
+use Brackup::Util qw/tempfile/;
 
-    my $n_files_done = 0;   # int
-    my @files;         # Brackup::File objs
+sub new {
+    my ($class, $parent, $backup_file) = @_;
+    my $self = bless {}, $class;
 
-    $self->debug("Discovering files in ", $root->path, "...\n");
-    $self->report_progress(0, "Discovering files in " . $root->path . "...");
-    $root->foreach_file(sub {
-        my ($file) = @_;  # a Brackup::File
-        push @files, $file;
-        $n_files++;
-        $n_kb += $file->size / 1024;
-    });
+    $self->{parent} = $parent;
+    $self->{backup_file} = $backup_file;
 
-    $self->debug("Number of files: $n_files\n");
-
-    # calc needed chunks
-    if ($ENV{CALC_NEEDED}) {
-        my $fn = 0;
-        foreach my $f (@files) {
-            $fn++;
-            if ($fn % 100 == 0) { warn "$fn / $n_files ...\n"; }
-            foreach my $pc ($f->chunks) {
-                if ($target->stored_chunk_from_inventory($pc)) {
-                    $pc->forget_chunkref;
-                    next;
-                }
-                $n_kb_up_need += $pc->length / 1024;
-                $pc->forget_chunkref;
-            }
+    $self->{stats} = Brackup::BackupStats->new;
+    $self->{comp_chunk} = undef;
+    $self->{cur_file} = undef;
+    $self->{metafh} = undef;
+    $self->{file_has_shown_status} = 0;
+    $self->{modecounts} = {}; # type -> mode(octal) -> count
+    $self->{stored_files} = ();
+    $self->{stored_chunks} = ();
+
+    return $self;
+}
+
+sub debug {
+    my ($self, @m) = @_;
+    return unless $self->{parent}->{verbose};
+    my $line = join("", @m);
+    chomp $line;
+    print $line, "\n";
+}
+
+sub report_progress {
+    my ($self, $percent, $message) = @_;
+
+    if ($self->{zenityprogress}) {
+        if (defined($message) && length($message) > 100) {
+            $message = substr($message, 0, 100)."...";
         }
-        warn "kb need to upload = $n_kb_up_need\n";
+        print STDOUT "#", $message, "\n" if defined $message;
+        print STDOUT $percent, "\n" if defined $percent;
     }
+}
 
+sub show_status {
+    my ($self) = @_;
+};
 
-    my $chunk_iterator = Brackup::ChunkIterator->new(@files);
+sub run {
+    my ($self) = @_;
 
-    my $gpg_iter;
-    my $gpg_pm;   # gpg ProcessManager
-    if ($gpg_rcpt) {
-        ($chunk_iterator, $gpg_iter) = $chunk_iterator->mux_into(2);
-        $gpg_pm = Brackup::GPGProcManager->new($gpg_iter, $target);
-    }
+    $self->_begin();
+    $self->_gather_chunks();
+    $self->_end();
+}
 
-    my $cur_file; # current (last seen) file
-    my @stored_chunks;
-    my $file_has_shown_status = 0;
-
-    my $end_file = sub {
-        return unless $cur_file;
-        $self->add_file($cur_file, [ @stored_chunks ]);
-        $n_files_done++;
-        $n_kb_done += $cur_file->size / 1024;
-        $cur_file = undef;
-    };
-    my $show_status = sub {
-        # use either size of files in normal case, or if we pre-calculated
-        # the size-to-upload (by looking in inventory, then we'll show the
-        # more accurate percentage)
-        my $percdone = 100 * ($n_kb_up_need ?
-                              ($n_kb_up / $n_kb_up_need) :
-                              ($n_kb_done / $n_kb));
-        my $mb_remain = ($n_kb_up_need ?
-                         ($n_kb_up_need - $n_kb_up) :
-                         ($n_kb - $n_kb_done)) / 1024;
-
-        $self->debug(sprintf("* %-60s %d/%d (%0.02f%%; remain: %0.01f MB)",
-                             $cur_file->path, $n_files_done, $n_files, $percdone,
-                             $mb_remain));
-
-        $self->report_progress($percdone);
-    };
-    my $start_file = sub {
-        $end_file->();
-        $cur_file = shift;
-        @stored_chunks = ();
-        $show_status->() if $cur_file->is_dir;
-        if ($gpg_iter) {
-            # catch our gpg iterator up.  we want it to be ahead of us,
-            # nothing iteresting is behind us.
-            $gpg_iter->next while $gpg_iter->behind_by > 1;
-        }
-        $file_has_shown_status = 0;
-    };
-
-    my $merge_under = $root->merge_files_under;
-    my $comp_chunk  = undef;
-
-    # records are either Brackup::File (for symlinks, directories, etc), or
-    # PositionedChunks, in which case the file can asked of the chunk
-    while (my $rec = $chunk_iterator->next) {
-        if ($rec->isa("Brackup::File")) {
-            $start_file->($rec);
-            next;
-        }
-        my $pchunk = $rec;
-        if ($pchunk->file != $cur_file) {
-            $start_file->($pchunk->file);
-        }
+sub stats {
+    my ($self) = @_;
 
-        # have we already stored this chunk before?  (iterative backup)
-        my $schunk;
-        if ($schunk = $target->stored_chunk_from_inventory($pchunk)) {
-            $pchunk->forget_chunkref;
-            push @stored_chunks, $schunk;
-            next;
-        }
+    return $self->{stats};
+}
+
+sub _begin {
+    my ($self) = @_;
+
+    $self->{metafh} = tempfile();
+}
 
-        # weird case... have we stored this same pchunk digest in the
-        # current comp_chunk we're building?  these aren't caught by
-        # the above inventory check, because chunks in a composite
-        # chunk aren't added to the inventory until after the the composite
-        # chunk has fully grown (because it's not until it's fully grown
-        # that we know the handle for it, its digest)
-        if ($comp_chunk && ($schunk = $comp_chunk->stored_chunk_from_dup_internal_raw($pchunk))) {
-            $pchunk->forget_chunkref;
-            push @stored_chunks, $schunk;
-            next;
+sub _emit_file {
+    my ($self, $file) = @_;
+
+    return undef unless $file;
+
+    my ($first, @chunks) = $file->chunks;
+    if ($first) {
+        $self->_emit_chunk($first);
+        foreach (@chunks) {
+            $self->_emit_chunk($_);
         }
+    } else {
+        # File without chunks
+        $self->_emit_chunk($file);
+    }
+}
 
-        $show_status->() unless $file_has_shown_status++;
-        $self->debug("  * storing chunk: ", $pchunk->as_string, "\n");
-        $self->report_progress(undef, $pchunk->file->path . " (" . $pchunk->offset . "," . $pchunk->length . ")");
+sub _emit_chunk {
+    my ($self, $rec) = @_;
+    my $merge_under = $self->{parent}->{root}->merge_files_under;
 
-        unless ($self->{dryrun}) {
-            $schunk = Brackup::StoredChunk->new($pchunk);
+    if ($rec->isa("Brackup::File")) {
+        $self->start_file($rec);
+        return;
+    }
 
-            # encrypt it
-            if ($gpg_rcpt) {
-                $schunk->set_encrypted_chunkref($gpg_pm->enc_chunkref_of($pchunk));
-            }
+    my $pchunk = $rec;
+    if ($pchunk->file != $self->{cur_file}) {
+        $self->start_file($pchunk->file);
+    }
 
-            # see if we should pack it into a bigger blob
-            my $chunk_size = $schunk->backup_length;
-
-            # see if we should merge this chunk (in this case, file) together with
-            # other small files we encountered earlier, into a "composite chunk",
-            # to be stored on the target in one go.
-
-            # Note: no technical reason for only merging small files (is_entire_file),
-            # and not the tails of larger files.  just don't like the idea of files being
-            # both split up (for big head) and also merged together (for little end).
-            # would rather just have 1 type of magic per file.  (split it or join it)
-            if ($merge_under && $chunk_size < $merge_under && $pchunk->is_entire_file) {
-                if ($comp_chunk && ! $comp_chunk->can_fit($chunk_size)) {
-                    $self->debug("Finalizing composite chunk $comp_chunk...");
-                    $comp_chunk->finalize;
-                    $comp_chunk = undef;
-                }
-                $comp_chunk ||= Brackup::CompositeChunk->new($root, $target);
-                $comp_chunk->append_little_chunk($schunk);
-            } else {
-                # store it regularly, as its own chunk on the target
-                $target->store_chunk($schunk)
-                    or die "Chunk storage failed.\n";
-                $target->add_to_inventory($pchunk => $schunk);
-            }
+    # have we already stored this chunk before?  (iterative backup)
+    my $schunk;
+    if ($schunk = $self->{parent}->{target}->stored_chunk_from_inventory($pchunk)) {
+        $pchunk->forget_chunkref;
+        push @{$self->{stored_chunks}}, $schunk;
+        return;
+    }
 
-            # if only this worked... (LWP protocol handler seems to
-            # get confused by its syscalls getting interrupted?)
-            #local $SIG{CHLD} = sub {
-            #    print "some child finished!\n";
-            #    $gpg_pm->start_some_processes;
-            #};
+    # weird case... have we stored this same pchunk digest in the
+    # current comp_chunk we're building?  these aren't caught by
+    # the above inventory check, because chunks in a composite
+    # chunk aren't added to the inventory until after the the composite
+    # chunk has fully grown (because it's not until it's fully grown
+    # that we know the handle for it, its digest)
+    if ($self->{comp_chunk} &&
+        ($schunk = $self->{comp_chunk}->stored_chunk_from_dup_internal_raw($pchunk))) {
+        $pchunk->forget_chunkref;
+        push @{$self->{stored_chunks}}, $schunk;
+        return;
+    }
 
+    $self->show_status() unless $self->{file_has_shown_status}++;
+    $self->debug("  * storing chunk: ", $pchunk->as_string, "\n");
+    $self->report_progress(undef, $pchunk->file->path . " (" .
+        $pchunk->offset . "," . $pchunk->length . ")");
 
-            $n_kb_up += $pchunk->length / 1024;
-            push @stored_chunks, $schunk;
+    unless ($self->{dryrun}) {
+        $schunk = Brackup::StoredChunk->new($pchunk);
+
+        # see if we should pack it into a bigger blob
+        my $chunk_size = $schunk->backup_length;
+
+        # see if we should merge this chunk (in this case, file) together with
+        # other small files we encountered earlier, into a "composite chunk",
+        # to be stored on the target in one go.
+
+        # Note: no technical reason for only merging small files (is_entire_file),
+        # and not the tails of larger files.  just don't like the idea of files being
+        # both split up (for big head) and also merged together (for little end).
+        # would rather just have 1 type of magic per file.  (split it or join it)
+        if ($merge_under && $chunk_size < $merge_under && $pchunk->is_entire_file) {
+            if ($self->{comp_chunk} && ! $self->{comp_chunk}->can_fit($chunk_size)) {
+                $self->debug('Finalizing composite chunk ', $self->{comp_chunk}, '...');
+                $self->{comp_chunk}->finalize;
+                $self->{comp_chunk} = undef;
+            }
+            $self->{comp_chunk} ||= Brackup::CompositeChunk->new($self->{parent}->{root},
+                                                                 $self->{parent}->{target});
+            $self->{comp_chunk}->append_little_chunk($schunk);
+        } else {
+            # store it regularly, as its own chunk on the target
+            $self->{parent}->{target}->store_chunk($schunk)
+                or die "Chunk storage failed.\n";
+            $self->{parent}->{target}->add_to_inventory($pchunk => $schunk);
         }
 
-        #$stats->note_stored_chunk($schunk);
-
-        # DEBUG: verify it got written correctly
-        if ($ENV{BRACKUP_PARANOID}) {
-            die "FIX UP TO NEW API";
-            #my $saved_ref = $target->load_chunk($handle);
-            #my $saved_len = length $$saved_ref;
-            #unless ($saved_len == $chunk->backup_length) {
-            #    warn "Saved length of $saved_len doesn't match our length of " . $chunk->backup_length . "\n";
-            #    die;
-            #}
+        push @{$self->{stored_chunks}}, $schunk;
+    }
+
+    $pchunk->forget_chunkref;
+    $schunk->forget_chunkref if $schunk;
+}
+
+sub start_file {
+    my ($self, $file) = @_;
+
+    $self->end_file();
+    $self->{cur_file} = $file;
+    $self->{stored_chunks} = ();
+    $self->show_status() if $self->{cur_file}->is_dir;
+
+    $self->{file_has_shown_status} = 0;
+}
+
+sub end_file {
+    my ($self) = @_;
+    my $file = $self->{cur_file};
+
+    return unless $file;
+
+    push @{$self->{stored_files}}, [$file, $self->{stored_chunks}];
+
+    $self->{modecounts}{$file->type}{$file->mode}++;
+    $self->{cur_file} = undef;
+}
+
+sub _write_meta {
+    my ($self) = @_;
+    my $metafh = $self->{metafh};
+
+    # Don't write anything while we're building a composite chunk
+    unless ($self->{comp_chunk}) {
+        foreach (@{$self->{stored_files}}) {
+            my ($file, @stored_chunks) = @{$_};
+            print $metafh $file->as_rfc822(@stored_chunks, $self);
         }
 
-        $pchunk->forget_chunkref;
-        $schunk->forget_chunkref if $schunk;
+        $self->{stored_files} = ();
+    }
+}
+
+sub _end {
+    my ($self) = @_;
+    my $metafh = $self->{metafh};
+
+    $self->end_file();
+
+    if ($self->{comp_chunk}) {
+        $self->{comp_chunk}->finalize;
+        $self->{comp_chunk} = undef;
     }
-    $end_file->();
-    $comp_chunk->finalize if $comp_chunk;
+
+    $self->_write_meta();
 
     unless ($self->{dryrun}) {
-        # write the metafile
+        my $backup_file = $self->{backup_file};
+
         $self->debug("Writing metafile ($backup_file)");
-        $self->report_progress(100, "Saving metafile " . $backup_file);
-        open (my $metafh, ">$backup_file") or die "Failed to open $backup_file for writing: $!\n";
-        print $metafh $self->backup_header;
-        $self->foreach_saved_file(sub {
-            my ($file, $schunk_list) = @_;
-            print $metafh $file->as_rfc822($schunk_list, $self);  # arrayref of StoredChunks
-        });
-        close $metafh or die;
-
-        my $contents;
-
-        # store the metafile, encrypted, on the target
-        if ($gpg_rcpt) {
-            my $encfile = $backup_file . ".enc";
-            system($self->{root}->gpg_path, $self->{root}->gpg_args,
-                   "--trust-model=always",
-                   "--recipient", $gpg_rcpt, "--encrypt", "--output=$encfile", "--yes", $backup_file)
-                and die "Failed to run gpg while encryping metafile: $!\n";
-            $contents = _contents_of($encfile);
-            unlink $encfile;
-        } else {
-            $contents = _contents_of($backup_file);
-        }
+        $self->report_progress(100, "Saving metafile $backup_file");
+
+        # Go to start of temporary file
+        seek $metafh, 0, 0;
+
+        # Concatenate metadata
+        my $contents = $self->backup_header;
+        $contents .= do { local $/; <$metafh>; };
+
+        open (my $out, '>', $backup_file)
+            or die "Failed to open $backup_file for writing: $!\n";
+        print $out $contents;
+        close $out;
 
         # store it on the target
-        $self->debug("Storing metafile to " . ref($target));
-        my $name = $self->{root}->publicname . "-" . $self->backup_time;
-        $target->store_backup_meta($name, $contents);
+        $self->debug("Storing metafile to ", ref($self->{parent}->{target}));
+        my $name = $self->{parent}->{root}->publicname . "-" . $self->{parent}->backup_time;
+        $self->{parent}->{target}->store_backup_meta($name, $contents);
     }
-    $self->report_progress(100, "Backup complete.");
 
-    return $stats;
-}
+    $self->report_progress(100, "Backup complete.");
 
-sub _contents_of {
-    my $file = shift;
-    open (my $fh, $file) or die "Failed to read contents of $file: $!\n";
-    return do { local $/; <$fh>; };
+    close $metafh or die $!;
 }
 
 sub default_file_mode {
@@ -292,18 +296,16 @@ sub _default_mode {
     return (sort { $map->{$b} <=> $map->{$a} } keys %$map)[0];
 }
 
-sub backup_time {
-    my $self = shift;
-    return $self->{backup_time} ||= time();
-}
-
 sub backup_header {
-    my $self = shift;
+    my ($self) = @_;
+    my $root = $self->{parent}->{root};
+    my $now = $self->{parent}->backup_time;
     my $ret = "";
-    my $now = $self->backup_time;
+
     $ret .= "BackupTime: " . $now . " (" . localtime($now) . ")\n";
-    $ret .= "BackupDriver: " . ref($self->{target}) . "\n";
-    if (my $fields = $self->{target}->backup_header) {
+    $ret .= "BackupDriver: " . ref($self->{parent}->{target}) . "\n";
+
+    if (my $fields = $self->{parent}->{target}->backup_header) {
         foreach my $k (keys %$fields) {
             die "Bogus header field from driver" unless $k =~ /^\w+$/;
             my $val = $fields->{$k};
@@ -311,49 +313,57 @@ sub backup_header {
             $ret .= "Driver-$k: $val\n";
         }
     }
-    $ret .= "RootName: " . $self->{root}->name . "\n";
-    $ret .= "RootPath: " . $self->{root}->path . "\n";
-    $ret .= "DefaultFileMode: " . $self->default_file_mode . "\n";
-    $ret .= "DefaultDirMode: " . $self->default_directory_mode . "\n";
-    if (my $rcpt = $self->{root}->gpg_rcpt) {
-        $ret .= "GPG-Recipient: $rcpt\n";
+
+    $ret .= "RootName: " . $root->name . "\n";
+    $ret .= "RootPath: " . $root->path . "\n";
+
+    if (defined($self->default_file_mode)) {
+        $ret .= "DefaultFileMode: " . $self->default_file_mode . "\n";
+    }
+
+    if (defined($self->default_directory_mode)) {
+        $ret .= "DefaultDirMode: " . $self->default_directory_mode . "\n";
     }
-    $ret .= "\n";
-    return $ret;
-}
 
-sub add_file {
-    my ($self, $file, $handlelist) = @_;
-    $self->{modecounts}{$file->type}{$file->mode}++;
-    push @{ $self->{saved_files} }, [ $file, $handlelist ];
+    return "$ret\n";
 }
 
-sub foreach_saved_file {
-    my ($self, $cb) = @_;
-    foreach my $rec (@{ $self->{saved_files} }) {
-        $cb->(@$rec);  # Brackup::File, arrayref of Brackup::StoredChunk
-    }
+
+package Brackup::Backup::_Sync;
+use strict;
+use warnings;
+use base 'Brackup::Backup::_Base';
+
+sub new {
+    my ($class, @params) = @_;
+    my $self = $class->SUPER::new(@params);
+
+    $self->{files} = [];
+
+    return $self;
 }
 
-sub debug {
-    my ($self, @m) = @_;
-    return unless $self->{verbose};
-    my $line = join("", @m);
-    chomp $line;
-    print $line, "\n";
+sub _begin {
+    my ($self) = @_;
+    my $root = $self->{parent}->{root};
+
+    $self->report_progress(0, "Discovering files in " . $root->path . "...");
+    $self->debug("Discovering files in ", $root->path, " ...\n");
+    $root->foreach_file(sub {
+        my ($file) = @_;  # a Brackup::File
+        push @{$self->{files}}, $file;
+    });
+    $self->debug("Number of files: ", scalar(@{$self->{files}}), "\n");
+
+    $self->SUPER::_begin();
 }
 
-sub report_progress {
-    my ($self, $percent, $message) = @_;
+sub _gather_chunks {
+    my ($self) = @_;
 
-    if ($self->{zenityprogress}) {
-        if (defined($message) && length($message) > 100) {
-            $message = substr($message, 0, 100)."...";
-        }
-        print STDOUT "#", $message, "\n" if defined $message;
-        print STDOUT $percent, "\n" if defined $percent;
+    foreach (@{$self->{files}}) {
+        $self->_emit_file($_);
     }
 }
 
 1;
-
-- 
1.5.5.3



More information about the brackup mailing list