Added support for git filter and inflate --all.

Göran Krampe
1 parent aaf9bcef
Showing 2 changed files with 139 additions and 54 deletions
blimp.nim
lapp.nim
-import md5, os, osproc, parseopt2, strutils, parsecfg, streams, lapp, subexes
+import md5, os, osproc, parseopt2, strutils, parsecfg, streams, lapp, subexes, tables
  
 # blimp is a little utility program for handling large files
 # in git repositories. Its inspired by git-fat and s3annex
@@ -11,7 +11,7 @@ import md5, os, osproc, parseopt2, strutils, parsecfg, streams, lapp, subexes
 # Use "blimp i mybigfile" to inflate it back to original size.
 #
 # When deflated the file only has:
-#   "hash:" <filename> + <an md5sum>
+#   "blimphash:" <filename> + <an md5sum>
 # ...inside it.
 #
 # The file is copied over to a local dir:
@@ -30,14 +30,14 @@ import md5, os, osproc, parseopt2, strutils, parsecfg, streams, lapp, subexes
  
 const 
   versionMajor* = 0
-  versionMinor* = 2
-  versionPatch* = 1
+  versionMinor* = 3
+  versionPatch* = 0
   versionAsString* = $versionMajor & "." & $versionMinor & "." & $versionPatch
  
 var
   blimpStore, remoteBlimpStore, uploadCommandFormat, downloadCommandFormat, deleteCommandFormat, rsyncPassword, blimpVersion: string = nil
   homeDir, currentDir, gitRootDir: string
-  verbose, stdio: bool
+  verbose, stdio, inflateAll: bool
   stdinContent: string = nil
  
 let
@@ -72,10 +72,11 @@ delete = &quot;rsync --password-file $3/.blimp.pass -dv --delete --existing --ignore-
 # version = 0.2 
 """
  
-# Find git root dir or nil
-proc gitRoot(): string =
+  
+
+proc cmd(cmd: string): string =
   try:
-    let tup = execCmdEx("git rev-parse --show-toplevel")
+    let tup = execCmdEx(cmd)
     if tup[1] == 0:
       result = strip(tup[0])
     else:
@@ -83,6 +84,29 @@ proc gitRoot(): string =
   except:
     result = nil
  
+# Find git root dir or nil
+proc gitRoot(): string =
+  cmd("git rev-parse --show-toplevel")
+
+# Git config
+proc gitConfigSet(key, val: string) =
+  discard cmd("git config " & $key & " " & $val)
+
+proc gitConfigGet(key: string): string =
+  cmd("git config --get " & $key)
+
+# Set blimp filter
+proc setBlimpFilter() =
+  gitConfigSet("filter.blimp.clean", "\"blimp -s d %f\"")
+  gitConfigSet("filter.blimp.smudge", "\"blimp -s i %f\"")
+
+
+# Ensure the filter is set
+proc ensureBlimpFilter() =
+  if gitConfigGet("filter.blimp.clean") != "":
+    setBlimpFilter()
+
+
 # Simple expansion of %home%, %cwd% and %gitroot%
 proc expandDirs(templ: string): string =
   result = templ.replace("%home%", homeDir)
@@ -96,8 +120,7 @@ proc parseConfFile(filename: string) =
   var f = newFileStream(filename, fmRead)
   if f != nil:
     if verbose: echo "Reading config: " & filename
-    var p: CfgParser
-    
+    var p: CfgParser  
     open(p, f, filename)
     while true:
       var e = next(p)
@@ -200,8 +223,8 @@ proc deleteFromBlimpStore(blimpFilename, filename: string) =
   remoteDeleteFile(blimpFilename)
  
 proc blimpFileNameFromString(line: string): string =
-  let hashline = split(strip(line), {':'})
-  if hashline[0] == "hash":
+  let hashline = split(strip(line), ':')
+  if hashline[0] == "blimphash":
     result = hashline[1]
   else:
     result = nil
@@ -236,11 +259,18 @@ proc deflate(filename: string) =
     blimpFilename = computeBlimpFilename(filename)
     copyToBlimpStore(filename, blimpFilename)
     if stdio:
-      write(stdout, "hash:" & blimpFilename)
+      write(stdout, "blimphash:" & blimpFilename)
     else:
-      writeFile(filename, "hash:" & blimpFilename)
+      writeFile(filename, "blimphash:" & blimpFilename)
     if verbose: echo("\t" & filename & " deflated.")
  
+# Iterator over all deflated files in the git clone
+iterator allDeflated() =
+  let filenames = cmd("git ls-files " & gitRootDir).split('\l')
+  for fn in filenames:
+    if not blimpFilename(fn).isNil:
+      yield fn
+
 # Parse out hash from hash stub and copy back original content from blimpStore.
 proc inflate(filename: string) =
   if verbose: echo "Inflating " & filename
@@ -290,41 +320,45 @@ proc dumpConfig() =
   echo "\tblimpVersion: " & $blimpVersion
   echo "\n"
  
-let help = """
+let synopsis = """
   blimp [options] <command> <filenames...>
     -h,--help                Show this
     --version                Show version of blimp
     -v,--verbose             Verbosity, only works without -s
-    -s,--stdio               If given, use stdin/stdout for content.
+    -i,--init                Set blimp filter in git config
+    ----------
     <command>   (string)     (d)eflate, (i)nflate, remove
-    <filenames> (string...)        One or more filepaths to inflate/deflate
-
+    -a,--all                 Operate on all deflated files in clone
+    ----------
+    -s,--stdio               If given, use stdin/stdout for content.
+    <filenames> (string...)  One or more filepaths to inflate/deflate
+"""
+let help = """
   blimp is a little utility program for handling large files
   in git repositories. Its inspired by git-fat and s3annex
   but doesn't rely on S3 for storage - it uses rsync like git-fat.
-  It is a single binary without any dependencies.
+  It is a single binary without any dependencies. Its not as advanced
+  as git-fat but basically does the same thing.
  
   Manual use:
  
-  Use "blimp d mybigfile" to deflate it before commit.
-  Use "blimp i mybigfile" to inflate it back to original size.
+  Use "blimp d mybigfile" to deflate a file, typically before commit.
+  Use "blimp i mybigfile" to inflate it back to original content.
  
   When deflated the file only has:
-    "hash:" <filename> "-" <md5sum>
+    "blimphash:" <filename> "-" <md5sum>
   ...inside it.
  
-  Deflate is run before you add the big file to the index for committing.
-  Deflate will replace the file contents with a hash, and copy the
-  real content to your local blimpstore:
+  Deflate also copies the real content to your local blimpstore:
  
-    "blimpstore"/<filename>-<md5sum>
+    <blimpstore>/<filename>-<md5sum>
  
-  ...and if configured also upload it to "remote", using rsync.
+  ...and if configured also uploads it to "remote", using rsync.
  
   Configuration is in these locations in order:
  
     ./.blimp.conf
-    "gitroot"/.blimp.conf
+    <gitroot>/.blimp.conf
     ~/<blimpstore>/.blimp.conf
     ~/.blimp.conf
  
@@ -337,18 +371,40 @@ let help = &quot;&quot;&quot;
   synced with a master rsync repository that is typically shared.
  
   Inflate will bring back the original content by copying from
-  your local blimpstore, and if its not there, first downloading from the remote.
+  your local blimpstore, and if its not there, first download from the remote.
   Use this whenever you need to work/edit the big file - in order to get
   its real content.
  
   The filenames given are all processed. If -s is used content is processed via
   stdin/stdout and only one filename can be passed. This is used when running blimp
-  via a git filter (smudge/clean).
+  via a git filter (smudge/clean), see below.
  
   The remove command (no single character shortcut) will remove the file(s) content
   both from the local blimpstore and from the remote. This only removes
   the current content version, not older versions. The file itself is first
   inflated, if needed, and not deleted. This only "unblimps" the file.
+  
+  In order to have blimp work automatically you can:
+  
+  * Create a .gitattributes file with lines like:
+    *.png filter=blimp -text
+    
+  * Configure blimp as a filter by running:
+    git config filter.blimp.clean "blimp -s d %f"
+    git config filter.blimp.smudge "blimp -s i %f"
+
+  The above git config can be done by running "blimp --init".
+
+  When the above is done (per clone) git will automatically run blimp deflate
+  just before committing and blimp inflate when operations are done.
+  
+  This means that if you clone a git repository that already has a .gitattributes
+  file in it that uses the blimp filter, then you should do:
+  
+    blimp --init inflate --all
+  
+  This will configure the blimp filter and also find and inflate all deflated
+  files throughout the clone.
 """
  
 ################################ main #####################################
@@ -359,9 +415,10 @@ currentDir = getCurrentDir()
 gitRootDir = gitRoot()
  
 # Using lapp to get args, on parsing failure this will show usage automatically
-var args = parse(help)
+var args = parse(synopsis)
 verbose = args["verbose"].asBool
 stdio = args["stdio"].asBool
+inflateAll = args["all"].asBool
  
 # Can't do verbose with -s, that messes up stdout,
 # read in all of stdin once and for all
@@ -376,7 +433,7 @@ if stdio:
  
 # Parse configuration files, may shadow and override each other
 parseConfFile(currentDir / ".blimp.conf")
-if not gitRootDir.isNil:
+if not gitRootDir.isNil and gitRootDir != currentDir:
   parseConfFile(gitRootDir / ".blimp.conf")
  
 # If we haven't gotten a blimpstore yet, we set a default one
@@ -387,35 +444,47 @@ if existsDir(blimpStore):
   parseConfFile(blimpStore / ".blimp.conf")
 parseConfFile(homeDir / ".blimp.conf")
  
+# Let's just show what we have :)
 if verbose: dumpConfig()
  
 # These two are special, they short out
-if args.showHelp: quit(help)
+if args.showHelp: quit(synopsis & help)
 if args.showVersion: quit("blimp version: " & versionAsString)
  
 # Check blimpVersion
 if not blimpVersion.isNil and blimpVersion != versionAsString:
   quit("Wrong version of blimp, configuration wants: " & blimpVersion)
  
+# Should we install the filter?
+if args["init"].asBool:
+  ensureBlimpFilter()
+  if verbose: echo("Installed blimp filter")
+
 let command = args["command"].asString
-let filenames = args["filenames"].asSeq
+var filenames: seq[PValue]
+if args.hasKey("filenames"):
+  filenames = args["filenames"].asSeq
  
 # Make sure the local blimpstore is setup.
 setupBlimpStore()
  
-
 # Do the deed
-if command == "d" or command == "deflate":
-  for fn in filenames:
-    deflate(fn.asString)
-elif command == "i" or command == "inflate":
-  for fn in filenames:
-    inflate(fn.asString)
-elif command == "remove":
-  for fn in filenames:
-    remove(fn.asString)
-else:
-  quit("Unknown command, only (d)eflate or (i)inflate are valid.", 6)
+if command != "":
+  if command == "d" or command == "deflate":
+    for fn in filenames:
+      deflate(fn.asString)
+  elif command == "i" or command == "inflate":
+    if inflateAll:
+      for fn in allDeflated():
+        inflate(fn)
+    else:
+      for fn in filenames:
+        inflate(fn.asString)
+  elif command == "remove":
+    for fn in filenames:
+      remove(fn.asString)
+  else:
+    quit("Unknown command, only (d)eflate or (i)inflate are valid.", 6)
  
 # All good
 quit(0)
@@ -17,6 +17,8 @@ type
     trange
     telipsis
     tchar
+    toption
+    tdivider
  
 proc thisChar(L: PLexer):char = L.str[L.idx]
 proc next(L: PLexer) = L.idx += 1
@@ -35,9 +37,15 @@ proc get(L: PLexer; t: var TLexType): string =
   t = tchar
   case c
   of '-':  # '-", "--"
+    t = toption
     if thisChar(L) == '-':
       result.add('-')
       next(L)
+      if thisChar(L) == '-': # "---..."
+        t = tdivider
+        result.add('-') 
+        while thisChar(L) == '-':
+          next(L)
   of Letters: # word
     t = tword
     while thisChar(L) in Letters:
@@ -112,14 +120,13 @@ proc intValue(v: int): PValue =   PValue(kind: vInt, asInt: v)
 proc floatValue(v: float): PValue = PValue(kind: vFloat, asFloat: v)
  
 proc seqValue(v: seq[PValue]): PValue = PValue(kind: vSeq, asSeq: v)
-
-const MAX_FILES = 30
  
 type
   PSpec = ref TSpec
   TSpec = object
     defVal: string
     ptype: string
+    group: int
     needsValue, multiple, used: bool
 var
   progname, usage: string
@@ -134,6 +141,7 @@ proc parseSpec(u: string) =
   var
     L: PLexer
     tok: string
+    groupCounter: int
     k = 1
  
   let lines = u.splitLines
@@ -173,7 +181,8 @@ proc parseSpec(u: string) =
       k += 1
       tok = L.get
       if tok != ">": fail("argument must be enclosed in <...>")
-        
+    elif tok == "---": # divider
+      inc(groupCounter)
     if getnext:  tok = L.get
     if tok == ":": # allowed to have colon after flags
       tok = L.get
@@ -205,8 +214,8 @@ proc parseSpec(u: string) =
       defValue = "false"
  
     if name != nil:
-      # echo("Param: " & name & " type: " & $ftype & " needsvalue: " & $(ftype != "bool") & " default: " & $defValue & " multiple: " & $multiple)
-      let spec = PSpec(defVal:defValue, ptype: ftype, needsValue: ftype != "bool",multiple:multiple)
+      #echo("Param: " & name & " type: " & $ftype & " group: " & $groupCounter & " needsvalue: " & $(ftype != "bool") & " default: " & $defValue & " multiple: " & $multiple)
+      let spec = PSpec(defVal:defValue, ptype: ftype, group: groupCounter, needsValue: ftype != "bool",multiple:multiple)
       aliases[alias] = name        
       parm_spec[name] = spec
  
@@ -297,12 +306,19 @@ proc parseArguments*(usage: string, args: seq[string]): Table[string,PValue] =
     if info.used:
       if flag == "help" or flag == "version":
         enableChecks = false
-  
-
+        
+  # Check maximum group used
+  var maxGroup = 0
+  for item in flagvalues:
+    info = get_spec(item[0])
+    if maxGroup < info.group:
+      maxGroup = info.group
+    
   # any flags not mentioned?
   for flag,info in parm_spec:
     if not info.used:
-      if info.defVal == "": # no default!
+      # Is there no default and we have used options in this group?
+      if info.defVal == "" and info.group <= maxGroup: 
         failures.add("required option or argument missing: " & flag)
       else:
         flagvalues.add(@[flag,info.defVal])