Added areas, moved to scp, added upload/download and various fixes.

Göran Krampe
1 parent 3e2f4c14
Showing 2 changed files with 150 additions and 101 deletions
blimp.nim
blimp.nimble
@@ -2,30 +2,49 @@ import md5, os, osproc, parseopt2, strutils, parsecfg, streams, lapp, subexes, t
 # blimp is a little utility program for handling large files
 # in git repositories. Its inspired by git-fat and s3annex
-# but doesn't rely on S3 for storage - it uses rsync like git-fat.
+# but doesn't rely on S3 for storage - it uses sftp but
+# performs the file operations using externally configured commands.
 # It is a single binary without any dependencies.
 #
+# It can also keep track of multiple remote areas and do up/downloads
+# independent from git. This is useful in build scripts, up and
+# downloading artifacts.
+#
 # Just run blimp --help for detailed help.
 const 
   versionMajor* = 0
-  versionMinor* = 3
+  versionMinor* = 4
   versionPatch* = 0
   versionAsString* = $versionMajor & "." & $versionMinor & "." & $versionPatch
+type
+  RemoteArea = ref object of RootObj
+    name*: string
+    url*: string
+    upload*: string
+    download*: string
+    delete*: string
+
 var
-  blimpStore, remoteBlimpStore, uploadCommandFormat, downloadCommandFormat, deleteCommandFormat, rsyncPassword, blimpVersion: string = nil
+  blimpStore, blimpVersion: string = nil
   homeDir, currentDir, gitRootDir: string
   verbose, stdio, onAllDeflated, onAllFiltered: bool
-  stdinContent: string = nil
+  stdinContent, area: string = nil
+  areas = newTable[string, RemoteArea]()
+  remoteArea: RemoteArea
 let
   defaultConfig = """
-[rsync]
+[blimp]
+# Minimal version, otherwise stop
+# version = """ & versionAsString & """
+
 # Set your local blimpstore directory. You can use %home%, %cwd% and %gitroot% in paths, works cross platform.
 # Example:
 #   # Place it inside the git clone
 #   blimpstore = "%gitroot%/.git/blimpstore"
+#
 #   # Place it in current working directory (not very useful)
 #   blimpstore = "%cwd%/blimpstore"
 #
@@ -33,27 +52,28 @@ let
 #   # Place it in the users home directory
 #   blimpstore = "%home%/blimpstore"
-# Set this to your remote rsync location
-remote = "blimpuser@some-rsync-server.com::blimpstore"
-# Set this to your rsync password, it will be written out as a password-file called .blimp.pass on every rsync.
-password = "some-good-rsync-password-for-blimpuser"
-
-# The following three formats should not need editing.
-# $1 is the blimp filename, $2 is remote location and $3 is the local blimpstore directory set above.
-# NOTE: The password-file .blimp.pass will be created by blimp on every command, do not remove that option.
-upload = "rsync --password-file $3/.blimp.pass -avzP $3/$1 $2/"
-download = "rsync --password-file $3/.blimp.pass -avzP $2/$1 $3/"
-# This deletes a single file from destination, that is already deleted in source. Yeah... insane! But it works.
-delete = "rsync --password-file $3/.blimp.pass -dv --delete --existing --ignore-existing --include '$1' --exclude '*' $3/ $2"
-
-[blimp]
-# Minimal version, otherwise stop
-# version = """ & versionAsString
+[areas]
+# This is where ypu define up/download areas. The area called "remote" is the
+# default area used unless --area is used.
+# $1 is the blimp filename, $2 is the remote url and $3 is the local blimpstore directory set above.
+remote-url = "blimpuser@some-server.com:/var/opt/blimpstore"
+remote-upload = "scp -pq $3/$1 $2/"
+remote-download = "scp -pq $2/$1 $3/"
+# Example area that can be used with upload/download commands and --area option.
+release-url = "blimpuser@some-server.com:/var/opt/release"
+release-upload = "scp -r $3/$1 $2/"
+release-download = "scp -r $2/$1 $3/"
+""" 
 proc cmd(cmd: string): string =
   try:
-    let tup = execCmdEx(cmd)
+    # Otherwise pipes will not work for git commands etc
+    when defined(windows):
+      let tup = execCmdEx("cmd /c \"" & cmd & "\"")
+    else:
+      let tup = execCmdEx(cmd) 
+    #echo "cmd: " & $cmd & "err:" & $tup[1]
     if tup[1] == 0:
       result = strip(tup[0])
     else:
@@ -99,31 +119,41 @@ proc parseConfFile(filename: string) =
     if verbose: echo "Reading config: " & filename
     var p: CfgParser  
     open(p, f, filename)
+    var section: string
+    var area: RemoteArea
     while true:
       var e = next(p)
       case e.kind
       of cfgEof: 
         break
       of cfgSectionStart:
-        continue # Ignore
+        section = e.section
       of cfgKeyValuePair:
-        case e.key
-        of "blimpstore":
-          if blimpStore.isNil: blimpStore = expandDirs(e.value)
-        of "remote":
-          if remoteBlimpStore.isNil: remoteBlimpStore = expandDirs(e.value)
-        of "password":
-          if rsyncPassword.isNil: rsyncPassword = e.value
-        of "upload":
-          if uploadCommandFormat.isNil: uploadCommandFormat = expandDirs(e.value)
-        of "download":
-          if downloadCommandFormat.isNil: downloadCommandFormat = expandDirs(e.value)
-        of "delete":
-          if deleteCommandFormat.isNil: deleteCommandFormat = expandDirs(e.value)
-        of "version":
-          if blimpVersion.isNil: blimpVersion = e.value
+        case section
+        of "blimp":
+          case e.key
+          of "blimpstore":
+            if blimpStore.isNil: blimpStore = e.value                  
+          of "version":
+            if blimpVersion.isNil: blimpVersion = e.value
+          else:
+            quit("Unknown blimp configuration: " & e.key)
         else:
-          quit("Unknown configuration: " & e.key)
+          # Then we presume its an area
+          if area.isNil or area.name != section:
+            area = RemoteArea(name: section)
+            areas[area.name] = area
+          case e.key
+          of "url":
+            if area.url.isNil: area.url = expandDirs(e.value)
+          of "upload":
+            if area.upload.isNil: area.upload = expandDirs(e.value)
+          of "download":
+            if area.download.isNil: area.download = expandDirs(e.value)
+          of "delete":
+            if area.delete.isNil: area.delete = expandDirs(e.value)
+          else:
+            quit("Unknown area configuration: " & e.key)
       of cfgOption:
         quit("Unknown configuration: " & e.key)
       of cfgError:
@@ -131,42 +161,35 @@ proc parseConfFile(filename: string) =
     close(p)
 # Trivial helper to enable verbose
-proc run(cmd: string): auto =
+proc run(cmd: string): int =
   if verbose: echo(cmd)
   execCmd(cmd)
-# Every rsync command, make sure we have a password file
-proc rsyncRun(cmd: string): auto =
-  if not rsyncPassword.isNil:
-    writeFile(blimpStore / ".blimp.pass", rsyncPassword)
-    if execCmd("chmod 600 " & blimpStore / ".blimp.pass") != 0:
-      quit("Failed to chmod 600 " & blimpStore / ".blimp.pass")
-  run(cmd)
-
-# Upload a file to the remote master blimpStore
-proc uploadFile(blimpFilename: string) =
-  if remoteBlimpStore.isNil:
-    echo("Remote blimpstore not set in configuration file, skipping uploading content:\n\t" & blimpFilename)
+
+# Upload a file to the remote area
+proc uploadFile(filename, fromDir: string) =
+  if remoteArea.isNil:
+    echo("Remote area not set in configuration file, skipping uploading content:\n\t" & filename)
     return
-  let errorCode = rsyncRun(format(uploadCommandFormat, blimpFilename, remoteBlimpStore, blimpStore))
+  let errorCode = run(format(remoteArea.upload, filename, remoteArea.url, fromDir))
   if errorCode != 0:
-    quit("Something went wrong uploading " & blimpFilename & " to " & remoteBlimpStore, 2)
+    quit("Something went wrong uploading " & filename & " to " & remoteArea.url, 2)
-# Download a file to the remote master blimpStore
-proc downloadFile(blimpFilename: string) =
-  if remoteBlimpStore.isNil:
-    quit("Remote blimpstore not set in configuration file, can not download content:\n\t" & blimpFilename)
-  let errorCode = rsyncRun(format(downloadCommandFormat, blimpFilename, remoteBlimpStore, blimpStore))
+# Download a file to the remote area
+proc downloadFile(filename, toDir: string) =
+  if remoteArea.isNil:
+    quit("Remote area not set in configuration file, can not download content:\n\t" & filename)
+  let errorCode = run(format(remoteArea.download, filename, remoteArea.url, toDir))
   if errorCode != 0:
-    quit("Something went wrong downloading " & blimpFilename & " from " & remoteBlimpStore, 3)
+    quit("Something went wrong downloading " & filename & " from " & remoteArea.url, 3)
 # Delete a file from the remote master blimpStore
-proc remoteDeleteFile(blimpFilename: string) =
-  if remoteBlimpStore.isNil:
+proc remoteDeleteFile(filename: string) =
+  if remoteArea.isNil:
     return
-  let errorCode = rsyncRun(format(deleteCommandFormat, blimpFilename, remoteBlimpStore, blimpStore))
+  let errorCode = run(format(remoteArea.delete, filename, remoteArea.url, blimpStore))
   if errorCode != 0:
-    quit("Something went wrong deleting " & blimpFilename & " from " & remoteBlimpStore, 3)
+    quit("Something went wrong deleting " & filename & " from " & remoteArea.url, 3)
 # Copy content to blimpStore and upload if it was a new file or upload == true.
 proc copyToBlimpStore(filename, blimpFilename: string) =
@@ -178,13 +201,13 @@ proc copyToBlimpStore(filename, blimpFilename: string) =
         quit("Failed writing file: " & blimpStore / blimpFilename & " from stdin", 1)
     else:
       copyFile(filename, blimpStore / blimpFilename)
-    uploadFile(blimpFilename)
+    uploadFile(blimpFilename, blimpStore)
 # Copy content from blimpStore, and downloading first if needed
 proc copyFromBlimpStore(blimpFilename, filename: string) =
   if not existsFile(blimpStore / blimpFilename):
-    downloadFile(blimpFilename)
+    downloadFile(blimpFilename, blimpStore)
   if stdio:
     try:
       var content = readFile(blimpStore / blimpFilename)
@@ -279,18 +302,29 @@ proc remove(filename: string) =
   echo("\t" & filename & " content removed from blimpstore locally and remotely.")
-# Copy original file to blimpStore and replace with hash stub in git.
-proc upload(filename: string) =
+# Make sure a file already in blimpstore is uploaded to remote area.
+proc push(filename: string) =
   if verbose: echo "Uploading " & filename
   var blimpFilename = blimpFilename(filename)
   if blimpFilename.isNil:
     blimpFilename = computeBlimpFilename(filename)
   if existsFile(blimpStore / blimpFilename):
-    uploadFile(blimpFilename)
+    uploadFile(blimpFilename, blimpStore)
     if verbose: echo("\t" & filename & " uploaded.")
   else:
     if verbose: echo("\t" & filename & " is not in blimpstore, skipping.")
+# Upload a file to a remote area
+proc upload(filename: string) =
+  if verbose: echo "Uploading " & filename
+  uploadFile(filename, currentDir)
+  if verbose: echo("\t" & filename & " uploaded.")
+    
+# Download a file from a remote area.
+proc download(filename: string) =
+  if verbose: echo "Downloading " & filename
+  downloadFile(filename, currentDir)
+  if verbose: echo("\t" & filename & " downloaded.")
 proc setupBlimpStore() =
   try:
@@ -311,27 +345,25 @@ proc `$`(x: string): string =
 proc dumpConfig() =
   echo "\nDump of configuration:"
   echo "\tblimpStore: " & blimpStore
-  echo "\tremoteBlimpStore: " & remoteBlimpStore
-  echo "\tuploadCommandFormat: " & uploadCommandFormat
-  echo "\tdownloadCommandFormat: " & downloadCommandFormat
-  echo "\tdeleteCommandFormat: " & deleteCommandFormat
-  echo "\trsyncPassword: " & $rsyncPassword
+  echo "\tremote-url: " & remoteArea.url
+  echo "\tremote-upload: " & remoteArea.upload
+  echo "\tremote-download: " & remoteArea.download
   echo "\tblimpVersion: " & $blimpVersion
   echo "\n"
 let synopsis = """
   blimp [options] <command> <filenames...>
-    -h,--help                Show this
-    --version                Show version of blimp
-    -v,--verbose             Verbosity, only works without -s
-    -i,--init                Set blimp filter in git config
+    -h,--help                  Show this
+    --version                  Show version of blimp
+    -v,--verbose               Verbosity, only works without -s
     ----------
-    <command>   (string)     (d)eflate, (i)nflate, remove, upload
-    -a,--all                 Operate on all deflated files in clone
-    -f,--filter              Operate on all files matching blimp filter
+    <command>  (string)        (d)eflate, (i)nflate, init, remove, push, upload, download
+    --all                      Operate on all deflated files in clone
+    -f,--filter                Operate on all files matching blimp filter
+    -a,--area (default remote) The area to use for remote up/downloads
     ----------
-    -s,--stdio               If given, use stdin/stdout for content.
-    <filenames> (string...)  One or more filepaths to inflate/deflate
+    -s,--stdio                 If given, use stdin/stdout for content.
+    <filenames> (string...)    One or more filepaths to inflate/deflate
 """
 let help = """
@@ -385,9 +417,12 @@ let help = &quot;&quot;&quot;
   the current content version, not older versions. The file itself is first
   inflated, if needed, and not deleted. This only "unblimps" the file.
-  The upload command (no single character shortcut) will upload the given file
+  The push command (no single character shortcut) will force upload the given file
   from the local blimpstore to the remote. This is normally done automatically,
-  but this way you can make sure they are up on the remote.
+  but this way you can make sure they are synced onto the remote.
+  
+  The upload and download commands are used to distribute artifacts typically in a
+  build script. If no --area is given, they use the standard "remote" area.
   In order to have blimp work automatically you can:
@@ -398,7 +433,7 @@ let help = &quot;&quot;&quot;
     git config filter.blimp.clean "blimp -s d %f"
     git config filter.blimp.smudge "blimp -s i %f"
-  The above git config can be done by running "blimp --init".
+  The above git config can be done by running "blimp init" just like git.
   When the above is done (per clone) git will automatically run blimp deflate
   just before committing and blimp inflate when operations are done.
@@ -406,9 +441,10 @@ let help = &quot;&quot;&quot;
   This means that if you clone a git repository that already has a .gitattributes
   file in it that uses the blimp filter, then you should do:
-    blimp --init inflate --filter
+    blimp init
+    blimp inflate --filter
-  This will configure the blimp filter and also find and inflate all deflated
+  This will configure the blimp filter and then find and inflate all deflated
   files throughout the clone.
 """
@@ -425,6 +461,7 @@ verbose = args[&quot;verbose&quot;].asBool
 stdio = args["stdio"].asBool
 onAllDeflated = args["all"].asBool
 onAllFiltered = args["filter"].asBool
+area = args["area"].asString
 # Can't do verbose with -s, that messes up stdout,
 # read in all of stdin once and for all
@@ -436,19 +473,23 @@ if stdio:
   except:
     quit("Failed reading stdin", 1)
-
 # Parse configuration files, may shadow and override each other
 parseConfFile(currentDir / ".blimp.conf")
 if not gitRootDir.isNil and gitRootDir != currentDir:
   parseConfFile(gitRootDir / ".blimp.conf")
+if existsDir(homeDir / "blimpstore"):
+  parseConfFile(homeDir / "blimpstore" / ".blimp.conf")
+
+parseConfFile(homeDir / ".blimp.conf")
+
 # If we haven't gotten a blimpstore yet, we set a default one
 if blimpStore.isNil:
   blimpStore = homeDir / "blimpstore"
-if existsDir(blimpStore):
-  parseConfFile(blimpStore / ".blimp.conf")
-parseConfFile(homeDir / ".blimp.conf")
+# Check if we have a configured remoteArea
+if areas.hasKey(area):
+  remoteArea = areas[area]
 # Let's just show what we have :)
 if verbose: dumpConfig()
@@ -461,12 +502,8 @@ if args.showVersion: quit(&quot;blimp version: &quot; &amp; versionAsString)
 if not blimpVersion.isNil and blimpVersion != versionAsString:
   quit("Wrong version of blimp, configuration wants: " & blimpVersion)
-# Should we install the filter?
-if args["init"].asBool:
-  ensureBlimpFilter()
-  if verbose: echo("Installed blimp filter")
-let command = args["command"].asString
+# Ok, let's see 
 var filenames = initSet[string]()
 # Add upp all files to operate on in a Set
@@ -484,9 +521,16 @@ if onAllFiltered:
 # Make sure the local blimpstore is setup.
 setupBlimpStore()
+
 # Do the deed
+let command = args["command"].asString
 if command != "":
-  if command == "d" or command == "deflate":
+  # Should we install the filter?
+  if command == "init":
+    ensureBlimpFilter()
+    echo("Installed blimp filter")
+    quit(0)
+  elif command == "d" or command == "deflate":
     for fn in filenames:
       deflate(fn)
   elif command == "i" or command == "inflate":
@@ -495,12 +539,17 @@ if command != &quot;&quot;:
   elif command == "remove":
     for fn in filenames:
       remove(fn)
+  elif command == "push":
+    for fn in filenames.items:
+      push(fn)
   elif command == "upload":
-    echo repr(filenames)
     for fn in filenames.items:
       upload(fn)
+  elif command == "download":
+    for fn in filenames.items:
+      download(fn)
   else:
-    quit("Unknown command: \"" & command & "\",  only (d)eflate, (i)inflate, remove or upload are valid.", 6)
+    quit("Unknown command: \"" & command & "\", use --help for valid commands.", 6)
 # All good
 quit(0)
 [Package]
 name          = "blimp"
-version       = "0.3"
+version       = "0.4"
 author        = "Göran Krampe"
 description   = "Utility that helps with big files in git, very similar to git-fat, s3annnex etc."
 license       = "MIT"