Upped to version 0.3, stdio supported so it can work as a git filter.

Göran Krampe
1 parent 78569137
Showing 2 changed files with 131 additions and 46 deletions
blimp.nim
blimp.nimble
@@ -2,25 +2,31 @@ import md5, os, osproc, parseopt2, strutils, parsecfg, streams, lapp, subexes
  
 # blimp is a little utility program for handling large files
 # in git repositories. Its inspired by git-fat and s3annex
-# but doesn't rely on S3 for storage, is a single binary without
-# need for Python, and has less features than git-fat. So far.
+# but doesn't rely on S3 for storage - it uses rsync like git-fat.
+# It is a single binary without any dependencies.
 #
 # Manual use:
 #
 # Use "blimp d mybigfile" to deflate it before commit.
 # Use "blimp i mybigfile" to inflate it back to original size.
 #
-# When deflated the file only has an md5sum string inside it.
+# When deflated the file only has:
+#   "hash:" <filename> + <an md5sum>
+# ...inside it.
 #
-# The file is copied over into:
-#  <homedir>/blimpStore/<originalfilename>-<md5sum>
+# The file is copied over to a local dir:
+#  <blimpstore>/<filename>-<md5sum>
 #
 # Configuration is in these locations in order:
 #
 #   ./.blimp.conf
 #   <gitroot>/.blimp.conf
-#   ~/blimpstore/.blimp.conf
+#   ~/<blimpstore>/.blimp.conf
 #   ~/.blimp.conf
+#
+# This way you can have settings per directory, per git clone,
+# per store and per user. A default blimpstore with a commented .blimp.conf
+# is created in ~/blimpstore if you run blimp and no .blimp.conf is found.
  
 const 
   versionMajor* = 0
@@ -29,14 +35,15 @@ const
   versionAsString* = $versionMajor & "." & $versionMinor & "." & $versionPatch
  
 var
-  blimpStore, remoteBlimpStore, uploadCommandFormat, downloadCommandFormat, deleteCommandFormat, rsyncPassword: string = nil
+  blimpStore, remoteBlimpStore, uploadCommandFormat, downloadCommandFormat, deleteCommandFormat, rsyncPassword, blimpVersion: string = nil
   homeDir, currentDir, gitRootDir: string
-  verbose: bool
+  verbose, stdio: bool
+  stdinContent: string = nil
  
 let
   defaultConfig = """
 [rsync]
-# Set your local blimpstore directory.
+# Set your local blimpstore directory. You can use %home%, %cwd% and %gitroot% in paths, works cross platform.
 # Example:
 #   # Place it inside the git clone
 #   blimpstore = "%gitroot%/.git/blimpstore"
@@ -49,17 +56,20 @@ let
  
 # Set this to your remote rsync location
 remote = "blimpuser@some-rsync-server.com::blimpstore"
-# Set this to your rsync password 
+# Set this to your rsync password, it will be written out as a password-file called .blimp.pass on every rsync.
 password = "some-good-rsync-password-for-blimpuser"
  
-# The following three formats should not need editing
-# $1 is the blimp filename, $2 is remote location and $3 is
-# the local blimpstore directory.
-# NOTE: The password-file will be created by blimp on every command.
+# The following three formats should not need editing.
+# $1 is the blimp filename, $2 is remote location and $3 is the local blimpstore directory set above.
+# NOTE: The password-file .blimp.pass will be created by blimp on every command, do not remove that option.
 upload = "rsync --password-file $3/.blimp.pass -avzP $3/$1 $2/"
 download = "rsync --password-file $3/.blimp.pass -avzP $2/$1 $3/"
 # This deletes a single file from destination, that is already deleted in source. Yeah... insane! But it works.
 delete = "rsync --password-file $3/.blimp.pass -dv --delete --existing --ignore-existing --include '$1' --exclude '*' $3/ $2"
+
+[blimp]
+# Minimal version, otherwise stop
+# version = 0.2 
 """
  
 # Find git root dir or nil
@@ -110,6 +120,8 @@ proc parseConfFile(filename: string) =
           if downloadCommandFormat.isNil: downloadCommandFormat = expandDirs(e.value)
         of "delete":
           if deleteCommandFormat.isNil: deleteCommandFormat = expandDirs(e.value)
+        of "version":
+          if blimpVersion.isNil: blimpVersion = e.value
         else:
           quit("Unknown configuration: " & e.key)
       of cfgOption:
@@ -159,14 +171,27 @@ proc remoteDeleteFile(blimpFilename: string) =
 # Copy content to blimpStore, no upload yet.
 proc copyToBlimpStore(filename, blimpFilename: string) =
   if not existsFile(blimpStore / blimpFilename):
-    copyFile(filename, blimpStore / blimpFilename)
-    uploadFile(blimpFilename)
+    if stdio:
+      try:
+        writeFile(blimpStore / blimpFilename, stdinContent)
+      except:
+        quit("Failed writing file: " & blimpStore / blimpFilename & " from stdin", 1)
+    else:
+      copyFile(filename, blimpStore / blimpFilename)
+      uploadFile(blimpFilename)
  
 # Copy content from blimpStore, and downloading first if needed
 proc copyFromBlimpStore(blimpFilename, filename: string) =
   if not existsFile(blimpStore / blimpFilename):
     downloadFile(blimpFilename)
-  copyFile(blimpStore / blimpFilename, filename)
+  if stdio:
+    try:
+      var content = readFile(blimpStore / blimpFilename)
+      write(stdout, content)
+    except:
+      quit("Failed reading file: " & blimpStore / blimpFilename & " to stdout", 1)
+  else:
+    copyFile(blimpStore / blimpFilename, filename)
  
 # Delete from blimpStore and remote.
 proc deleteFromBlimpStore(blimpFilename, filename: string) =
@@ -174,17 +199,23 @@ proc deleteFromBlimpStore(blimpFilename, filename: string) =
     removeFile(blimpStore / blimpFilename)
   remoteDeleteFile(blimpFilename)
  
-# Pick out blimpFilename (filename & "-" & hash)
-proc blimpFileName(filename: string): string=
-  var hashfile: File
-  if not open(hashfile, filename):
-    quit("Failed opening file: " & filename, 4)
-  let hashline = split(string(readLine(hashfile)), {':'})
+proc blimpFileNameFromString(line: string): string =
+  let hashline = split(strip(line), {':'})
   if hashline[0] == "hash":
     result = hashline[1]
   else:
     result = nil
  
+# Pick out blimpFilename (filename & "-" & hash)
+proc blimpFileName(filename: string): string =
+  if stdio:
+    blimpFileNameFromString(stdinContent)
+  else:
+    var hashfile: File
+    if not open(hashfile, filename):
+      quit("Failed opening file: " & filename, 4)
+    blimpFileNameFromString(string(readLine(hashfile)))  
+
 # Get hash and compute blimpFilename
 proc computeBlimpFilename(filename: string): string =
   var content: string
@@ -204,7 +235,10 @@ proc deflate(filename: string) =
   else:
     blimpFilename = computeBlimpFilename(filename)
     copyToBlimpStore(filename, blimpFilename)
-    writeFile(filename, "hash:" & blimpFilename)
+    if stdio:
+      write(stdout, "hash:" & blimpFilename)
+    else:
+      writeFile(filename, "hash:" & blimpFilename)
     if verbose: echo("\t" & filename & " deflated.")
  
 # Parse out hash from hash stub and copy back original content from blimpStore.
@@ -253,34 +287,69 @@ proc dumpConfig() =
   echo "\tdownloadCommandFormat: " & downloadCommandFormat
   echo "\tdeleteCommandFormat: " & deleteCommandFormat
   echo "\trsyncPassword: " & $rsyncPassword
+  echo "\tblimpVersion: " & $blimpVersion
   echo "\n"
  
 let help = """
   blimp [options] <command> <filenames...>
     -h,--help                Show this
     --version                Show version of blimp
-    -v,--verbose             Verbosity
+    -v,--verbose             Verbosity, only works without -s
+    -s,--stdio               If given, use stdin/stdout for content.
     <command>   (string)     (d)eflate, (i)nflate, remove
-    <filenames> (string...)  One or more filepaths to inflate/deflate
-    
-    Edit ~/blimpstore/.blimp.conf or <gitroot>/.blimp.conf and set a proper
-    remote and the proper rsync password to use.
-    
-    Deflate is run before you add the big file to the index for committing.
-    Deflate will replace the file contents with a hash, and copy the
-    real content to your local blimpstore, and if configured also upload it to
-    remote, using rsync.
-    
-    Inflate will bring back the original content by copying from
-    your local blimpstore, and if its not there, first downloading from the remote.
-    Use this whenever you need to work/edit the big file - in order to get
-    its real content.
-    
-    Remove (no single character shortcut) will remove the file(s) content
-    both from the local blimpstore and from the remote. This only removes
-    the current content version, not older versions. The file itself is first
-    inflated, if needed, and not deleted. This only "unblimps" the file.
-  """
+    <filenames> (string...)        One or more filepaths to inflate/deflate
+
+  blimp is a little utility program for handling large files
+  in git repositories. Its inspired by git-fat and s3annex
+  but doesn't rely on S3 for storage - it uses rsync like git-fat.
+  It is a single binary without any dependencies.
+
+  Manual use:
+
+  Use "blimp d mybigfile" to deflate it before commit.
+  Use "blimp i mybigfile" to inflate it back to original size.
+
+  When deflated the file only has:
+    "hash:" <filename> "-" <md5sum>
+  ...inside it.
+
+  Deflate is run before you add the big file to the index for committing.
+  Deflate will replace the file contents with a hash, and copy the
+  real content to your local blimpstore:
+  
+    "blimpstore"/<filename>-<md5sum>
+
+  ...and if configured also upload it to "remote", using rsync.
+
+  Configuration is in these locations in order:
+ 
+    ./.blimp.conf
+    "gitroot"/.blimp.conf
+    ~/<blimpstore>/.blimp.conf
+    ~/.blimp.conf
+
+  This way you can have settings per directory, per git clone,
+  per store and per user. A default blimpstore with a commented .blimp.conf
+  is created in ~/blimpstore if you run blimp and no .blimp.conf is found.
+
+  Edit ~/blimpstore/.blimp.conf (or in another location) and set a proper
+  remote and the proper rsync password. This ensures that its also properly
+  synced with a master rsync repository that is typically shared.
+
+  Inflate will bring back the original content by copying from
+  your local blimpstore, and if its not there, first downloading from the remote.
+  Use this whenever you need to work/edit the big file - in order to get
+  its real content.
+
+  The filenames given are all processed. If -s is used content is processed via
+  stdin/stdout and only one filename can be passed. This is used when running blimp
+  via a git filter (smudge/clean).
+
+  The remove command (no single character shortcut) will remove the file(s) content
+  both from the local blimpstore and from the remote. This only removes
+  the current content version, not older versions. The file itself is first
+  inflated, if needed, and not deleted. This only "unblimps" the file.
+"""
  
 ################################ main #####################################
 # Set some dirs
@@ -292,6 +361,18 @@ gitRootDir = gitRoot()
 # Using lapp to get args, on parsing failure this will show usage automatically
 var args = parse(help)
 verbose = args["verbose"].asBool
+stdio = args["stdio"].asBool
+
+# Can't do verbose with -s, that messes up stdout,
+# read in all of stdin once and for all
+if stdio:
+  verbose = false
+  try:
+    stdinContent = readAll(stdin)
+    close(stdin)
+  except:
+    quit("Failed reading stdin", 1)
+
  
 # Parse configuration files, may shadow and override each other
 parseConfFile(currentDir / ".blimp.conf")
@@ -312,6 +393,10 @@ if verbose: dumpConfig()
 if args.showHelp: quit(help)
 if args.showVersion: quit("blimp version: " & versionAsString)
  
+# Check blimpVersion
+if not blimpVersion.isNil and blimpVersion != versionAsString:
+  quit("Wrong version of blimp, configuration wants: " & blimpVersion)
+
 let command = args["command"].asString
 let filenames = args["filenames"].asSeq
  
 [Package]
 name          = "blimp"
-version       = "0.2"
+version       = "0.3"
 author        = "Göran Krampe"
 description   = "Utility that helps with big files in git, very similar to git-fat, s3annnex etc."
 license       = "MIT"