by Matthew Caylor Presteps:
backup_user ALL = NOPASSWD:/usr/bin/rsync,/bin/chown
Script Files (Incremental): This form creates multiple copies of the back up over time while only storing as few physical copies of each file as possible through the use of hard links. Rsync is used to copy files over ssh intelligently and unlink hard copies if a file is updated. The back up has at least two script files. We use the following:
All files are saved to $HOME/bin ======== make_snapshot.sh (Hourly) ==================================== #!/bin/bash # ---------------------------------------------------------------------- # mikes handy rotating-filesystem-snapshot utility # ---------------------------------------------------------------------- # this needs to be a lot more general, but the basic idea is it makes # rotating backup-snapshots of /home whenever called # ---------------------------------------------------------------------- # edited by Matthew Caylor for use in Dr. Xu's lab. Feb 9th, 2011 # ---------------------------------------------------------------------- # The concept is that the host this runs on backs up the remote host’s files # that way it is easier to rotate back ups. Data is pulled from remote host and # stored locally. # ---------------------------------------------------------------------- unset PATH # suggestion from H. Milz: avoid accidental use of $PATH # ------------- system commands used by this script -------------------- ID=/usr/bin/id; # This is to check to make sure the correct user is using the script. ECHO=/bin/echo; RM=/bin/rm; MV=/bin/mv; CP=/bin/cp; TOUCH=/bin/touch; MKDIR=/bin/mkdir; SUDO=/usr/bin/sudo; CHOWN=/bin/chown; CHMOD=/bin/chmod; FIND=/usr/bin/find; SSHAGENT=/usr/bin/ssh-agent SSHADD=/usr/bin/ssh-add KILL=/bin/kill RSYNC=/usr/bin/rsync; # ------------- file locations ----------------------------------------- REMOTE_HOST=<Remote Server you are backing up on local server>; SNAPSHOT_RW=<The base directory you are backing up to>; EXCLUDES=$HOME/settings/backup_exclude; #You can add to this file as you want. # See man rsync for details. DESTINATION=/<subdirectory for files>; SOURCE=<location on remote host that you are backing up>/; #The trailing / is important. # ------------- Passwordless SSH -------------------------------------- eval `$SSHAGENT` # Start ssh-agent $SSHADD # Connect to the ssh-agent # ------------- the script itself ------------------------------------------ # make sure we're running as the back up user if (( `$ID -u` != <user id> )); then { $ECHO "Sorry, must be <user>. Exiting..."; exit; } fi # make sure target directory exists if [ ! -d $SNAPSHOT_RW$DESTINATION ] ; then $MKDIR -p $SNAPSHOT_RW$DESTINATION; fi # rotating snapshots of the directory being backed up (fixme: this should be more general) # You can change the number of back up copies by changing steps 1 and 2. # step 1: delete the oldest snapshot, if it exists: if [ -d $SNAPSHOT_RW$DESTINATION/hourly.3 ] ; then \ $RM -rf $SNAPSHOT_RW$DESTINATION/hourly.3 ; \ fi ; # step 2: shift the middle snapshots(s) back by one, if they exist if [ -d $SNAPSHOT_RW$DESTINATION/hourly.2 ] ; then \ $MV $SNAPSHOT_RW$DESTINATION/hourly.2 $SNAPSHOT_RW$DESTINATION/hourly.3 ; \ fi; if [ -d $SNAPSHOT_RW$DESTINATION/hourly.1 ] ; then \ $MV $SNAPSHOT_RW$DESTINATION/hourly.1 $SNAPSHOT_RW$DESTINATION/hourly.2 ; \ fi; # step 3: make a hard-link-only (except for dirs) copy of the latest snapshot, # if that exists # as a note, this does not take up any extra space on the hard drive since it is all hard links. if [ -d $SNAPSHOT_RW$DESTINATION/hourly.0 ] ; then \ $CP -al $SNAPSHOT_RW$DESTINATION/hourly.0 $SNAPSHOT_RW$DESTINATION/hourly.1 ; \ fi; # step 4: rsync from the system into the latest snapshot (notice that # rsync behaves like cp --remove-destination by default, so the destination # is unlinked first. If it were not so, this would copy over the other # snapshot(s) too! $RSYNC \ -va --delete --delete-excluded \ --exclude-from="$EXCLUDES" \ --rsync-path="sudo rsync" \ -e ssh \ $REMOTE_HOST:$SOURCE $SNAPSHOT_RW$DESTINATION/hourly.0 ; # step 5: Modify the ownership and atributes. # As is, this will make ALL files backed up PUBLIC to all users on the system. $SUDO $CHOWN -R backup_user $SNAPSHOT_RW$DESTINATION/hourly.0 $FIND $SNAPSHOT_RW$DESTINATION/hourly.0 -type d -exec $CHMOD 755 {} \; $FIND $SNAPSHOT_RW$DESTINATION/hourly.0 -type f -exec $CHMOD +r {} \; # step 6: update the mtime of hourly.0 to reflect the snapshot time $TOUCH $SNAPSHOT_RW$DESTINATION/hourly.0 ; # and thats it for home. # ------------- Kill Passwordless SSH ---------------------------------- $KILL $SSH_AGENT_PID #kill the ssh agent. ========= daily_snapshot_rotate.sh ==================================== #!/bin/bash # ---------------------------------------------------------------------- # mikes handy rotating-filesystem-snapshot utility: daily snapshots # ---------------------------------------------------------------------- # intended to be run daily as a cron job when hourly.3 contains the # midnight (or whenever you want) snapshot; say, 13:00 for 4-hour snapshots. # ---------------------------------------------------------------------- unset PATH # ------------- system commands used by this script -------------------- ID=/usr/bin/id; ECHO=/bin/echo; RM=/bin/rm; MV=/bin/mv; CP=/bin/cp; # ------------- file locations ----------------------------------------- SNAPSHOT_RW=<The base directory you are backing up to>; SOURCE=/<subdirectory for files>; # A point of note, SOURCE here is DESTINATION in make_snapshot.sh # ------------- the script itself -------------------------------------- # make sure we're running as back up user if (( `$ID -u` != <user id> )); then { $ECHO "Sorry, must be <back up user>. Exiting..."; exit; } fi # step 1: delete the oldest snapshot, if it exists: if [ -d $SNAPSHOT_RW$SOURCE/daily.2 ] ; then \ $RM -rf $SNAPSHOT_RW$SOURCE/daily.2 ; \ fi ; # step 2: shift the middle snapshots(s) back by one, if they exist if [ -d $SNAPSHOT_RW$SOURCE/daily.1 ] ; then \ $MV $SNAPSHOT_RW$SOURCE/daily.1 $SNAPSHOT_RW$SOURCE/daily.2 ; \ fi; # step 3: shift daily.0 to daily.1 if [ -d $SNAPSHOT_RW$SOURCE/daily.0 ] ; then \ $MV $SNAPSHOT_RW$SOURCE/daily.0 $SNAPSHOT_RW$SOURCE/daily.1; \ fi; # step 3: make a hard-link-only (except for dirs) copy of # hourly.3, assuming that exists, into daily.0 if [ -d $SNAPSHOT_RW$SOURCE/hourly.3 ] ; then \ $CP -al $SNAPSHOT_RW$SOURCE/hourly.3 $SNAPSHOT_RW$SOURCE/daily.0 ; \ fi; # note: do *not* update the mtime of daily.0; it will reflect # when hourly.3 was made, which should be correct. ======== weeky_snapshot_rotate.sh ===================================== #!/bin/bash # ---------------------------------------------------------------------- # mikes handy rotating-filesystem-snapshot utility: daily snapshots # ---------------------------------------------------------------------- # intended to be run daily as a cron job when hourly.3 contains the # midnight (or whenever you want) snapshot; say, 13:00 for 4-hour snapshots. # ---------------------------------------------------------------------- unset PATH # ------------- system commands used by this script -------------------- ID=/usr/bin/id; ECHO=/bin/echo; RM=/bin/rm; MV=/bin/mv; CP=/bin/cp; # ------------- file locations ----------------------------------------- SNAPSHOT_RW=<The base directory you are backing up to>; SOURCE=/<subdirectory for files>; # A point of note, SOURCE here is DESTINATION in make_snapshot.sh # ------------- the script itself -------------------------------------- # make sure we're running as backup user if (( `$ID -u` != <backup user id> )); then { $ECHO "Sorry, must be <back up user>. Exiting..."; exit; } fi # step 1: delete the oldest snapshot, if it exists: if [ -d $SNAPSHOT_RW$SOURCE/weekly.0 ] ; then \ $RM -rf $SNAPSHOT_RW$SOURCE/weekly.0 ; \ fi ; # step 2: make a hard-link-only (except for dirs) copy of # daily.0, assuming that exists, into weekly.0 if [ -d $SNAPSHOT_RW$SOURCE/daily.2 ] ; then \ $CP -al $SNAPSHOT_RW$SOURCE/daily.2 $SNAPSHOT_RW$SOURCE/weekly.0 ; \ fi; # note: do *not* update the mtime of weekly.0; it will reflect Script Files (Single Stage): This back up option only keeps one back up copy but still uses rsync to intelligently copy the files over ssh. ======== single_stage.sh ============================================ #!/bin/bash # ---------------------------------------------------------------------- # mikes handy single-stage-snapshot utility # ---------------------------------------------------------------------- # this needs to be a lot more general, but the basic idea is it makes # rotating backup-snapshots of /home whenever called # ---------------------------------------------------------------------- # edited by Matthew Caylor for use in Dr. Xu's lab. Feb 9th, 2011 # ---------------------------------------------------------------------- # The concept is that the host this runs on backs up the remote host # that way you can more easily rotate back ups. # ---------------------------------------------------------------------- unset PATH # suggestion from H. Milz: avoid accidental use of $PATH # ------------- system commands used by this script ----------- ID=/usr/bin/id; ECHO=/bin/echo; RM=/bin/rm; MV=/bin/mv; CP=/bin/cp; TOUCH=/bin/touch; MKDIR=/bin/mkdir; SUDO=/usr/bin/sudo; CHOWN=/bin/chown; CHMOD=/bin/chmod; FIND=/usr/bin/find; SSHAGENT=/usr/bin/ssh-agent SSHADD=/usr/bin/ssh-add KILL=/bin/kill RSYNC=/usr/bin/rsync; # ------------- file locations ----------------------------------------- REMOTE_HOST=<Remote Server you are backing up on local server>; SNAPSHOT_RW=<The base directory you are backing up to>; EXCLUDES=$HOME/settings/backup_exclude; #You can add to this file as you want. # See man rsync for details. DESTINATION=/<subdirectory for files>; SOURCE=<location on remote host that you are backing up>/; #The trailing / is important. # ------------- Passwordless SSH -------------------------------------- eval `$SSHAGENT` $SSHADD # ------------- the script itself ------------------------------------------- # make sure we're running as root if (( `$ID -u` != <back up user id> )); then { $ECHO "Sorry, must be <back up user>. Exiting..."; exit; } fi # make sure target directory exists if [ ! -d $SNAPSHOT_RW$DESTINATION ] ; then $MKDIR -p $SNAPSHOT_RW$DESTINATION; fi # step 1: rsync from the system into the latest snapshot (notice that # rsync behaves like cp --remove-destination by default, so the destination # is unlinked first. If it were not so, this would copy over the other # snapshot(s) too! $RSYNC \ -va --delete --delete-excluded \ --exclude-from="$EXCLUDES" \ --rsync-path="sudo rsync" \ -e ssh \ $REMOTE_HOST:$SOURCE $SNAPSHOT_RW$DESTINATION ; # step 2: Modify the ownership and atributes. $SUDO $CHOWN -R backup_user $SNAPSHOT_RW$DESTINATION/ $FIND $SNAPSHOT_RW$DESTINATION/ -type d -exec $CHMOD 755 {} \; $FIND $SNAPSHOT_RW$DESTINATION/ -type f -exec $CHMOD +r {} \; # step 3: update the mtime to reflect the snapshot time $TOUCH $SNAPSHOT_RW$DESTINATION; # and thats it for home. # ------------- Kill Passwordless SSH ---------------------------------- $KILL $SSH_AGENT_PID Post Instructions You will need to add the relevant scripts to the crontab in order to get the back ups to run automatically. You can learn more about crontab by typing man crontab. The set up we use is below. Incremental:
Single-Stage:
These run as listed and record any output to the log folder in the back up user’s home directory sorted by date and update type. notes:
|
|
Wiki >
SSH-RSYNC Backup Set Upposted Mar 16, 2011, 6:57 PM by Dong Xu |