PERL   51

backup

Guest on 22nd September 2022 12:36:33 AM

  1. #!/usr/bin/perl
  2. #H# Kevin's rsync based backup system version 0.21
  3. #H#
  4. #H# This is a backup system that uses rsync as the primary tool.
  5. #H#
  6. #H# command line switches:
  7. #H# name                backup name (this is a directory containing configuration files)
  8. #H# -v|--verbose        be verbose
  9. #H#
  10. #H# See the included readme.txt file for more information including the format
  11. #H# of the configuration directories and files.
  12.  
  13. # default settings
  14. $BackupRoot="undefined";
  15. $BackupScripts=`pwd`;
  16. chomp $BackupScripts;
  17. $BackupSettings="$BackupScripts/settings";
  18. $DefaultNumberOldBackups=30;
  19. $HumanReadableOutput="yes";
  20. $BackupACLs="no";
  21. $BackupXATTRs="no";
  22. $UseNetworkCompression="no";
  23. $UseSSHIdentyFile="none";
  24. $ForceChecksum="no";
  25. $BackupIsFAT="no";
  26. $BackupIsNAS="no";
  27. $CompensateForFAT="no";
  28. $UseFilterFiles="no";
  29. $TransferWholeFiles="no";
  30. $UpdateFilesInPlace="yes";
  31. $WriteFilesSparsely="no";
  32. $IgnoreHardLinks="no";
  33. $ArchiveMethod="link-dest";
  34. $ExtraRsyncParams="";
  35. $HostLevelSubvolumes="yes";
  36. $Verbose="no";
  37. $PurgeOnly="no";
  38. $BackupName="error";
  39. $RmParams="-f";
  40. $MvParams="-f";
  41. $BTRFSRedir=">/dev/null";
  42. $LinkDest="";
  43.  
  44. # parse command line
  45. foreach $Arg (@ARGV) {
  46.   chomp $Arg;
  47.   if ($Arg eq "-v" || $Arg eq "--verbose") { $Verbose="yes"; }
  48.   if ($Arg =~ /^\w/) { $BackupName=$Arg; }
  49.   if ($Arg eq "-h" || $Arg eq "--help" || $Arg eq "help") {
  50.     open (SELF, $0);
  51.     while (<SELF>) {
  52.       if ($_ =~ /^#H# /) {
  53.         $_ =~ s/^#H# //;
  54.         print $_;
  55.       } # if help line
  56.     } # while help input
  57.     exit
  58.   } # if asking for help
  59. } # foreach args
  60.  
  61. # setup initial rsync parameters (these will be heavily modified by settings later
  62. $RsyncParams="--archive --one-file-system --hard-links --human-readable --inplace --numeric-ids --delete --delete-excluded";
  63. if ($Verbose eq "yes") {
  64.   $RsyncParams="$RsyncParams --verbose --itemize-changes --progress";
  65.   $RmParams="$RmParams -v";
  66.   $MvParams="$MvParams -v";
  67.   $BTRFSRedir="";
  68.   print "Running backup named $BackupName.\n";
  69. } # if verbose
  70.  
  71. # make sure a backup name was specified
  72. if ($BackupName eq "error") {
  73.   print "Error: Cannot find backup name on command line.\n";
  74.   exec ("$0 --help");
  75. } # if error
  76.  
  77. # verify that settings exist
  78. $Self=$0;
  79. chomp $Self;
  80. $Self =~ s/.*\///;
  81. $Self = "$BackupScripts/$Self";
  82. open (SELF, $Self) || die "ERROR: Cannot find backup scripts ($Self).\n";
  83. close (SELF);
  84. if (-d "$BackupSettings")  { } else {
  85.   die "ERROR: Cannot find backup settings ($BackupSettings)\n";
  86. } # if else backup settings exists
  87.  
  88. # verify the named backup
  89. if (-d "$BackupScripts/$BackupName") {
  90.   if (-f "$BackupScripts/$BackupName/backuptab") {
  91.   } else {
  92.     die "ERROR: No backuptab exists for $BackupName.\n";
  93.   }
  94. } else {
  95.   die "ERROR: No backup named $BackupName exists.\n";
  96. }
  97.  
  98. # read settings from files
  99. if (-f "$BackupSettings/settings.pl") { do "$BackupSettings/settings.pl"; }
  100. if (-f "$BackupScripts/$BackupName/settings.pl") { do "$BackupScripts/$BackupName/settings.pl"; }
  101.  
  102. # verify that settings are still sane...
  103. if (-d "$BackupRoot") { } else { die "ERROR: The Backup Root ($BackupRoot) does not exist.\n"; }
  104. if ($UseNetworkCompression eq "yes") { $RsyncParams="$RsyncParams --compress"; }
  105. if ($HumanReadableOutput eq "no") { $RsyncParams =~ s/--human-readable//g; }
  106. if ($BackupACLs eq "yes") { $RsyncParams = "$RsyncParams --acls"; }
  107. if ($BackupXATTRs eq "yes") { $RsyncParams = "$RsyncParams --xattrs"; }
  108. if ($UseSSHIdentyFile ne "none") {
  109.   if (-f "$UseSSHIdentyFile") {
  110.     $RsyncParams="$RsyncParams -e 'ssh -i $UseSSHIdentyFile'";
  111.   } else {
  112.     die "ERROR: You specified an ssh identity file that does not exist ($UseSSHIdentityFile).\n";
  113.   }
  114. }
  115.  
  116. if ($BackupIsNAS eq "yes") {
  117.   # Optimize for writing backups to an NFS mount.  Note that this is a really inneficient thing to do.
  118.   $TransferWholeFiles="yes";
  119.   $WriteFilesSparsely="no";
  120.   $UpdateFilesInPlace="no";
  121. }
  122.  
  123. if ($TransferWholeFiles eq "yes") { $RsyncParams= "$RsyncParams --whole-file"; }
  124. if ($WriteFilesSparsely eq "yes") {
  125.   $RsyncParams="$RsyncParams --sparse";
  126.   $UpdateFilesInPlace="no";
  127. }
  128. if ($UpdateFilesInPlace eq "no") { $RsyncParams =~ s/--inplace//g; }
  129. if ($IgnoreHardLinks eq "yes") { $RsyncParams =~ s/--hard-links//g; }
  130. if ($ForceChecksum eq "yes") { $RsyncParams="$RsyncParams --checksum"; }
  131. if ($BackupIsFAT eq "yes") {
  132.   print "Notice: The FAT filesystem is extremely limited.\n";
  133.   print "The FAT filesystem does not support any archiving method, file ownerships,\n";
  134.   print "hard links, symbolic links, or consistent timestamps.\n";
  135.   print "Therefore it is inappropriate for use as a backup device.\n";
  136.   print "The FAT filesystem is only useful for mirroring other FAT filesystems\n";
  137.   print "which is beyond the scope of this program.\n";
  138.   die "ERROR: Attempted to backup to FAT filesystem.\n";
  139. }
  140. if ($CompensateForFAT eq "yes") {
  141.   # we are backing up a FAT filesystem which has unstable timestamps
  142.   $RsyncParams="$RsyncParams --modify-window=3602";
  143. }
  144. if ($UseFilterFiles eq "yes") { $RsyncParams="$RsyncParams -F"; }
  145. if ($ArchiveMethod ne "link-dest") {
  146.   if ($ArchiveMethod ne "zfs") {
  147.     if ($ArchiveMethod ne "btrfs") {
  148.       die "ERROR: Unknown archive method ($ArchiveMethod) specified.\n";
  149.     }
  150.   }
  151. }
  152. if ($ExtraRsyncParams ne "") { $RsyncParams="$ExtraRsyncParams $RsyncParams"; }
  153.  
  154. # add in include and exclude files...
  155. if (-f "$BackupSettings/includes") { $RsyncParams="$RsyncParams --include-from=$BackupSettings/includes"; }
  156. if (-f "$BackupSettings/excludes") { $RsyncParams="$RsyncParams --exclude-from=$BackupSettings/excludes"; }
  157. if (-f "$BackupScripts/$BackupName/includes") { $RsyncParams="$RsyncParams --include-from=$BackupScripts/$BackupName/includes"; }
  158. if (-f "$BackupScripts/$BackupName/excludes") { $RsyncParams="$RsyncParams --exclude-from=$BackupScripts/$BackupName/excludes"; }
  159.  
  160. # gather information about other systems.
  161. # This allows you to get things like partition tables that are not stored in files.
  162. if (-f "$BackupScripts/$BackupName/infotab") {
  163.   if ($Verbose eq "yes") {
  164.     system ("getinfo.pl $BackupScripts/$BackupName/infotab --verbose");
  165.   } else {
  166.     system ("getinfo.pl $BackupScripts/$BackupName/infotab");
  167.   }
  168. }
  169.  
  170. # time to do some backups
  171. open (BACKUPTAB, "$BackupScripts/$BackupName/backuptab");
  172. while (<BACKUPTAB>) {
  173.   $TabLine=$_;
  174.   chomp $TabLine;
  175.   $TabLine =~ s/#.*//;
  176.   if ($TabLine !~ /^$/) {
  177.     $Host=$TabLine;
  178.     $Host =~ s/:.*//g;
  179.     $BackupString=$TabLine;
  180.     $BackupString =~ s/$Host://;
  181.     if ($BackupString =~ /^\//) {
  182.       # I see some kind of path to backup
  183.       ($SourcePath, $NumArchivesWanted)=split (/:/,$BackupString,2);
  184.       if ($NumArchivesWanted eq "d") { $NumArchivesWanted=$DefaultNumberOldBackups; }
  185.       if ($SourcePath eq "//") {
  186.         # I see the special case //
  187.         $RsyncParams =~ s/--one-file-system//g;
  188.         $SourcePath="/";
  189.       } # if //
  190.       if ($SourcePath eq "/*") {
  191.         # I see the special case /*
  192.         # The smart way to handle the special case * backup is to generate a temporary
  193.         # backup based on the output of df then rerun this script using that backup.
  194.         $TempBackupName=`mktemp --tmpdir=. --directory`;
  195.         chomp $TempBackupName;
  196.         open (DIRLIST, "ssh $Host df -lTP |");
  197.         open (TEMPBACKUPTAB, "> $TempBackupName/backuptab");
  198.         while (<DIRLIST>) {
  199.           $DFLine=$_;
  200.           chomp $DFLine;
  201.           if ($DFLine =~ /^\//) {
  202.             if ($DFLine !~ /tmpfs/) {
  203.               if ($DFLine !~ /iso9660/) {
  204.                 if ($DFLine !~ /cd9660/) {
  205.                   if ($DFLine !~ /squashfs/) {
  206.                     @DF=split ($DFLine);
  207.                     foreach $Df (@DF) {
  208.                       $FS=$Df;
  209.                     }
  210.                     chomp $FS;
  211.                     print (TEMPBACKUPTAB "$Host:$FS:$NumArchivesWanted\n");
  212.                   }
  213.                 }
  214.               }
  215.             }
  216.           }
  217.         }
  218.         close (DIRLIST);
  219.         close (TEMPBACKUPTAB);
  220.         $Params="";
  221.         if ($Verbose eq "yes") { $Params="$Params --verbose"; }
  222.         system ("$0 $Params $TempBackupName");
  223.         system ("rm -rf $TempBackupName");
  224.       # end of /*
  225.       } else {
  226.         # I have a regular path (or // has become /)
  227.         $LocalTestHostName=$Host;
  228.         $LocalTestHostName =~ s/\..*//;
  229.         $RemoteTestHostName=`ssh -x $Host hostname -s`;
  230.         chomp $RemoteTestHostName;
  231.         if ($LocalTestHostName ne $RemoteTestHostName) {
  232.           print "WARNING: Skipping $SourcePath on $Host because I cannot verify ssh access to $Host ($LocalTestHostName ne $RemoteTestHostName).\n";
  233.         } else {
  234.           # do the archive
  235.           if ($ArchiveMethod eq "zfs") { die "ZFS snapshot archiving not yet implamented.\n"; }
  236.           # determine the backup path with _ instead of /
  237.           # FSP = Fixed Source Path
  238.           $FSP=$SourcePath;
  239.           $FSP =~ s/\//_/g;
  240.           $Date=`date +'%Y-%m-%d.%H-%M-%S'`;
  241.           chomp $Date;
  242.           $FFSP="$FSP.$Date";
  243.           $FQFSP="$BackupRoot/$Host/$FFSP";
  244.           # determine current backup
  245.           $CurrentBackup="none";
  246.           if (-l "$BackupRoot/$Host/$FSP.current") {
  247.             $CurrentBackup=`readlink $BackupRoot/$Host/$FSP.current`;
  248.             chomp $CurrentBackup;
  249.             $CurrentBackup =~ s/.*\///g;
  250.           }
  251.           # handle special case 0
  252.           if ($NumArchivesWanted == 0) {
  253.             # not archiving anything
  254.             if ($CurrentBackup ne "none") {
  255.               if ($ArchiveMethod eq "link-dest") {
  256.                 # if a backup already exists we just rename it
  257.                 system ("rm $RmParams $BackupRoot/$Host/$FSP.current");
  258.                 system ("mv $MvParams $BackupRoot/$Host/$CurrentBackup $BackupRoot/$Host/$FSP.incomplete");
  259.               }
  260.             } else {
  261.               print "No existing backup found for $Host:$SourcePath.  Making a new backup.\n";
  262.               if ($ArchiveMethod eq "link-dest") {
  263.                 system ("mkdir -p $BackupRoot/$Host/$FSP.incomplete");
  264.               } elsif ($ArchiveMethod eq "btrfs") {
  265.                 if (-d "$BackupRoot/$Host") { }
  266.                 else {
  267.                   if ($HostLevelSubvolumes eq "yes") {
  268.                     system ("btrfs subvolume create $BackupRoot/$Host $BTRFSRedir")==0
  269.                       or die ("ERROR: Error creating btrfs subvolume $BackupRoot/$Host ($?).\n");
  270.                   } else {
  271.                     system ("mkdir $BackupRoot/$Host");
  272.                   }
  273.                 }
  274.                 if (-d "$BackupRoot/$Host/$FSP") { }
  275.                 else {
  276.                   system ("btrfs subvolume create $BackupRoot/$Host/$FSP $BTRFSRedir")==0
  277.                     or die ("Error creating btrfs subvolume $BackupRoot/$Host/$FSP ($?).\n");
  278.                 }
  279.               }
  280.             }
  281.           } else {
  282.             # determine if old backups exist
  283.             if ($CurrentBackup eq "none") {
  284.               if ($Verbose eq "yes") { print "No existing backups for $Host:$SourcePath.  Making a new one.\n"; }
  285.               if ($ArchiveMethod eq "link-dest" ) {
  286.                 system ("mkdir -p $BackupRoot/$Host/$FSP.incomplete");
  287.               } elsif ($ArchiveMethod eq "btrfs") {
  288.                 if (-d "$BackupRoot/$Host") { }
  289.                 else {
  290.                   if ($HostLevelSubvolumes eq "yes") {
  291.                     system ("btrfs subvolume create $BackupRoot/$Host $BTRFSRedir")==0
  292.                       or die ("ERROR: Error creating btrfs subvolume $BackupRoot/$Host ($?).\n");
  293.                   } else {
  294.                     system ("mkdir $BackupRoot/$Host");
  295.                   }
  296.                 }
  297.                 if (-d "$BackupRoot/$Host/$FSP") { }
  298.                 else {
  299.                   system ("btrfs subvolume create $BackupRoot/$Host/$FSP $BTRFSRedir")==0
  300.                     or die ("Error creating btrfs subvolume $BackupRoot/$Host/$FSP ($?).\n");
  301.                 }
  302.               }
  303.             } else {
  304.               # determine how many existing backups there are
  305.               @ BackupList=();
  306.               open (BACKUPLIST, "ls -d $BackupRoot/$Host/$FSP.* |");
  307.               while (<BACKUPLIST>) {
  308.                 $Data=$_;
  309.                 chomp $Data;
  310.                 if ($Data !~ /complete$/) {
  311.                   push (@BackupList,$Data);
  312.                 }
  313.               }
  314.               close (BACKUPLIST);
  315.               sort (@BackupList);
  316.               $NumOldBackups=$#BackupList;
  317.               if ($Verbose eq "yes") { print "There are $NumOldBackups old backups of $Host:$SourcePath ($NumArchivesWanted are wanted).\n"; }
  318.               #$NumOldBackups++;
  319.               if ($NumOldBackups > $NumArchivesWanted) {
  320.                 $TooManyBackups=$NumOldBackups-$NumArchivesWanted;
  321.                 if ($Verbose eq "yes") { print "There are $TooManyBackups too many old backups that need to be purged.\n"; }
  322.                 $DeleteCount=0;
  323.  
  324.                 while ($DeleteCount != $TooManyBackups) {
  325.                   if ($BackupList[$DeleteCount] =~ /.ToBePurged$/) {
  326.                     if ($Verbose eq "yes") { print "$BackupList[$DeleteCount] is already tagged for purging.  Are you not using the Purge program?\n"; }
  327.                   } else {
  328.                     if ($Verbose eq "yes") { print "Purging $BackupList[$DeleteCount]...\n"; }
  329.                     if ($ArchiveMethod eq "link-dest" || $ArchiveMethod eq "btrfs") {
  330.                       $TempDeleteName="$BackupList[$DeleteCount].ToBePurged";
  331.                       system ("mv $MvParams $BackupList[$DeleteCount] $TempDeleteName");
  332.                     #} elsif ($ArchiveMethod eq "btrfs") {
  333.                       #system ("btrfs subvolume delete $BackupList[$DeleteCount] $BTRFSRedir")==0
  334.                         #or print ("Warning: unable to purge old subvolume $BackupList[$DeleteCount] ($?).\n");
  335.                     }
  336.                   }
  337.                   $DeleteCount++;
  338.                 }
  339.  
  340.               }
  341.               # setup archive from current (link-dest or cp)
  342.               if ($ArchiveMethod eq "link-dest") {
  343.                 if ($Verbose eq "yes") { print "Linking to old backup $CurrentBackup for new backup.\n"; }
  344.                 system ("mkdir -p $BackupRoot/$Host/$FSP.incomplete");
  345.                 $LinkDest="--link-dest=$BackupRoot/$Host/$CurrentBackup";
  346.               }
  347.               if ($ArchiveMethod eq "cp") {
  348.                 if ($Verbose eq "yes") { print "Making duplicate of $CurrentBackup for the new backup\n"; }
  349.                 system ("mkdir -p $BackupRoot/$Host/$FSP.incomplete");
  350.                 # this is the time consuming part that makes the cp method suck.
  351.                 system ("cp -al $BackupRoot/$Host/$Current $BackupRoot/$Host/$FSP.incomplete")==0
  352.                   or die ("ERROR: cp -al failed ($?).\n");
  353.               }
  354.             }
  355.           }
  356.           # do the backup
  357.           if ($ArchiveMethod eq "link-dest" || $ArchiveMethod eq "cp") {
  358.             $BackupTarget="$FSP.incomplete";
  359.           } else {
  360.             $BackupTarget="$FSP";
  361.           }
  362.           if ($Verbose eq "yes") {
  363.             print ("Running: rsync $RsyncParams $Host:$SourcePath/ $BackupRoot/$Host/$BackupTarget/\n");
  364.           }
  365.           system ("rsync $RsyncParams $LinkDest $Host:$SourcePath/ $BackupRoot/$Host/$BackupTarget/") == 0
  366.             or print "WARNING: There was a problem backing up $Host:$SourcePath ($?).\n";
  367.           if ($? == 65280) { die "ALERT: Rsync was aborted (^C).\n"; }
  368.           if ($? == 65281) { die "ALERT: Rsync was aborted (^C).\n"; }
  369.           if ($ArchiveMethod eq "link-dest" || $ArchiveMethod eq "cp") {
  370.             system ("mv $MvParams $BackupRoot/$Host/$FSP.incomplete $FQFSP");
  371.           } else {
  372.             system ("btrfs subvolume snapshot $BackupRoot/$Host/$FSP $BackupRoot/$Host/$FSP.$Date $BTRFSRedir")==0
  373.               or die ("ERROR snapshotting btrfs subvolume $BackupRoot/$Host/$FSP to $BackupRoot/$Host/$FSP.$Date ($?).\n");
  374.           }
  375.           system ("rm -f $BackupRoot/$Host/$FSP.current");
  376.           system ("ln -sf $FFSP $BackupRoot/$Host/$FSP.current");
  377.         }
  378.       }
  379.     }
  380.     if ($BackupString =~ /^r!/) {
  381.       # I see a remote command to run
  382.       $RemoteCommand=$BackupString;
  383.       $RemoteCommand =~ s/^r!//;
  384.       system (qq[ssh $Host "$RemoteCommand"]);
  385.     }
  386.     if ($BackupString =~ /^l!/) {
  387.       # I see a local command to run
  388.       $LocalCommand=$BackupString;
  389.       $LocalCommand =~ s/^l!//;
  390.       system ("$LocalCommand")==0
  391.         or print "Warning: local command failed ($?).\n";
  392.     }
  393.   }
  394. }

Raw Paste


Login or Register to edit or fork this paste. It's free.