From 386557e646a97464c713aa6ee607008c85f37aad Mon Sep 17 00:00:00 2001 From: Göran Krampe Date: Tue, 18 Nov 2014 23:28:09 +0100 Subject: [PATCH] Added support for git filter and inflate --all. --- blimp.nim | 161 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------------------------------------------- lapp.nim | 32 ++++++++++++++++++++++++-------- 2 files changed, 139 insertions(+), 54 deletions(-) diff --git a/blimp.nim b/blimp.nim index 34913ae..d929f29 100644 --- a/blimp.nim +++ b/blimp.nim @@ -1,4 +1,4 @@ -import md5, os, osproc, parseopt2, strutils, parsecfg, streams, lapp, subexes +import md5, os, osproc, parseopt2, strutils, parsecfg, streams, lapp, subexes, tables # blimp is a little utility program for handling large files # in git repositories. Its inspired by git-fat and s3annex @@ -11,7 +11,7 @@ import md5, os, osproc, parseopt2, strutils, parsecfg, streams, lapp, subexes # Use "blimp i mybigfile" to inflate it back to original size. # # When deflated the file only has: -# "hash:" + +# "blimphash:" + # ...inside it. # # The file is copied over to a local dir: @@ -30,14 +30,14 @@ import md5, os, osproc, parseopt2, strutils, parsecfg, streams, lapp, subexes const versionMajor* = 0 - versionMinor* = 2 - versionPatch* = 1 + versionMinor* = 3 + versionPatch* = 0 versionAsString* = $versionMajor & "." & $versionMinor & "." & $versionPatch var blimpStore, remoteBlimpStore, uploadCommandFormat, downloadCommandFormat, deleteCommandFormat, rsyncPassword, blimpVersion: string = nil homeDir, currentDir, gitRootDir: string - verbose, stdio: bool + verbose, stdio, inflateAll: bool stdinContent: string = nil let @@ -72,10 +72,11 @@ delete = "rsync --password-file $3/.blimp.pass -dv --delete --existing --ignore- # version = 0.2 """ -# Find git root dir or nil -proc gitRoot(): string = + + +proc cmd(cmd: string): string = try: - let tup = execCmdEx("git rev-parse --show-toplevel") + let tup = execCmdEx(cmd) if tup[1] == 0: result = strip(tup[0]) else: @@ -83,6 +84,29 @@ proc gitRoot(): string = except: result = nil +# Find git root dir or nil +proc gitRoot(): string = + cmd("git rev-parse --show-toplevel") + +# Git config +proc gitConfigSet(key, val: string) = + discard cmd("git config " & $key & " " & $val) + +proc gitConfigGet(key: string): string = + cmd("git config --get " & $key) + +# Set blimp filter +proc setBlimpFilter() = + gitConfigSet("filter.blimp.clean", "\"blimp -s d %f\"") + gitConfigSet("filter.blimp.smudge", "\"blimp -s i %f\"") + + +# Ensure the filter is set +proc ensureBlimpFilter() = + if gitConfigGet("filter.blimp.clean") != "": + setBlimpFilter() + + # Simple expansion of %home%, %cwd% and %gitroot% proc expandDirs(templ: string): string = result = templ.replace("%home%", homeDir) @@ -96,8 +120,7 @@ proc parseConfFile(filename: string) = var f = newFileStream(filename, fmRead) if f != nil: if verbose: echo "Reading config: " & filename - var p: CfgParser - + var p: CfgParser open(p, f, filename) while true: var e = next(p) @@ -200,8 +223,8 @@ proc deleteFromBlimpStore(blimpFilename, filename: string) = remoteDeleteFile(blimpFilename) proc blimpFileNameFromString(line: string): string = - let hashline = split(strip(line), {':'}) - if hashline[0] == "hash": + let hashline = split(strip(line), ':') + if hashline[0] == "blimphash": result = hashline[1] else: result = nil @@ -236,11 +259,18 @@ proc deflate(filename: string) = blimpFilename = computeBlimpFilename(filename) copyToBlimpStore(filename, blimpFilename) if stdio: - write(stdout, "hash:" & blimpFilename) + write(stdout, "blimphash:" & blimpFilename) else: - writeFile(filename, "hash:" & blimpFilename) + writeFile(filename, "blimphash:" & blimpFilename) if verbose: echo("\t" & filename & " deflated.") +# Iterator over all deflated files in the git clone +iterator allDeflated() = + let filenames = cmd("git ls-files " & gitRootDir).split('\l') + for fn in filenames: + if not blimpFilename(fn).isNil: + yield fn + # Parse out hash from hash stub and copy back original content from blimpStore. proc inflate(filename: string) = if verbose: echo "Inflating " & filename @@ -290,41 +320,45 @@ proc dumpConfig() = echo "\tblimpVersion: " & $blimpVersion echo "\n" -let help = """ +let synopsis = """ blimp [options] -h,--help Show this --version Show version of blimp -v,--verbose Verbosity, only works without -s - -s,--stdio If given, use stdin/stdout for content. + -i,--init Set blimp filter in git config + ---------- (string) (d)eflate, (i)nflate, remove - (string...) One or more filepaths to inflate/deflate - + -a,--all Operate on all deflated files in clone + ---------- + -s,--stdio If given, use stdin/stdout for content. + (string...) One or more filepaths to inflate/deflate +""" +let help = """ blimp is a little utility program for handling large files in git repositories. Its inspired by git-fat and s3annex but doesn't rely on S3 for storage - it uses rsync like git-fat. - It is a single binary without any dependencies. + It is a single binary without any dependencies. Its not as advanced + as git-fat but basically does the same thing. Manual use: - Use "blimp d mybigfile" to deflate it before commit. - Use "blimp i mybigfile" to inflate it back to original size. + Use "blimp d mybigfile" to deflate a file, typically before commit. + Use "blimp i mybigfile" to inflate it back to original content. When deflated the file only has: - "hash:" "-" + "blimphash:" "-" ...inside it. - Deflate is run before you add the big file to the index for committing. - Deflate will replace the file contents with a hash, and copy the - real content to your local blimpstore: + Deflate also copies the real content to your local blimpstore: - "blimpstore"/- + /- - ...and if configured also upload it to "remote", using rsync. + ...and if configured also uploads it to "remote", using rsync. Configuration is in these locations in order: ./.blimp.conf - "gitroot"/.blimp.conf + /.blimp.conf ~//.blimp.conf ~/.blimp.conf @@ -337,18 +371,40 @@ let help = """ synced with a master rsync repository that is typically shared. Inflate will bring back the original content by copying from - your local blimpstore, and if its not there, first downloading from the remote. + your local blimpstore, and if its not there, first download from the remote. Use this whenever you need to work/edit the big file - in order to get its real content. The filenames given are all processed. If -s is used content is processed via stdin/stdout and only one filename can be passed. This is used when running blimp - via a git filter (smudge/clean). + via a git filter (smudge/clean), see below. The remove command (no single character shortcut) will remove the file(s) content both from the local blimpstore and from the remote. This only removes the current content version, not older versions. The file itself is first inflated, if needed, and not deleted. This only "unblimps" the file. + + In order to have blimp work automatically you can: + + * Create a .gitattributes file with lines like: + *.png filter=blimp -text + + * Configure blimp as a filter by running: + git config filter.blimp.clean "blimp -s d %f" + git config filter.blimp.smudge "blimp -s i %f" + + The above git config can be done by running "blimp --init". + + When the above is done (per clone) git will automatically run blimp deflate + just before committing and blimp inflate when operations are done. + + This means that if you clone a git repository that already has a .gitattributes + file in it that uses the blimp filter, then you should do: + + blimp --init inflate --all + + This will configure the blimp filter and also find and inflate all deflated + files throughout the clone. """ ################################ main ##################################### @@ -359,9 +415,10 @@ currentDir = getCurrentDir() gitRootDir = gitRoot() # Using lapp to get args, on parsing failure this will show usage automatically -var args = parse(help) +var args = parse(synopsis) verbose = args["verbose"].asBool stdio = args["stdio"].asBool +inflateAll = args["all"].asBool # Can't do verbose with -s, that messes up stdout, # read in all of stdin once and for all @@ -376,7 +433,7 @@ if stdio: # Parse configuration files, may shadow and override each other parseConfFile(currentDir / ".blimp.conf") -if not gitRootDir.isNil: +if not gitRootDir.isNil and gitRootDir != currentDir: parseConfFile(gitRootDir / ".blimp.conf") # If we haven't gotten a blimpstore yet, we set a default one @@ -387,35 +444,47 @@ if existsDir(blimpStore): parseConfFile(blimpStore / ".blimp.conf") parseConfFile(homeDir / ".blimp.conf") +# Let's just show what we have :) if verbose: dumpConfig() # These two are special, they short out -if args.showHelp: quit(help) +if args.showHelp: quit(synopsis & help) if args.showVersion: quit("blimp version: " & versionAsString) # Check blimpVersion if not blimpVersion.isNil and blimpVersion != versionAsString: quit("Wrong version of blimp, configuration wants: " & blimpVersion) +# Should we install the filter? +if args["init"].asBool: + ensureBlimpFilter() + if verbose: echo("Installed blimp filter") + let command = args["command"].asString -let filenames = args["filenames"].asSeq +var filenames: seq[PValue] +if args.hasKey("filenames"): + filenames = args["filenames"].asSeq # Make sure the local blimpstore is setup. setupBlimpStore() - # Do the deed -if command == "d" or command == "deflate": - for fn in filenames: - deflate(fn.asString) -elif command == "i" or command == "inflate": - for fn in filenames: - inflate(fn.asString) -elif command == "remove": - for fn in filenames: - remove(fn.asString) -else: - quit("Unknown command, only (d)eflate or (i)inflate are valid.", 6) +if command != "": + if command == "d" or command == "deflate": + for fn in filenames: + deflate(fn.asString) + elif command == "i" or command == "inflate": + if inflateAll: + for fn in allDeflated(): + inflate(fn) + else: + for fn in filenames: + inflate(fn.asString) + elif command == "remove": + for fn in filenames: + remove(fn.asString) + else: + quit("Unknown command, only (d)eflate or (i)inflate are valid.", 6) # All good quit(0) diff --git a/lapp.nim b/lapp.nim index 16cc9e5..0aa9d35 100644 --- a/lapp.nim +++ b/lapp.nim @@ -17,6 +17,8 @@ type trange telipsis tchar + toption + tdivider proc thisChar(L: PLexer):char = L.str[L.idx] proc next(L: PLexer) = L.idx += 1 @@ -35,9 +37,15 @@ proc get(L: PLexer; t: var TLexType): string = t = tchar case c of '-': # '-", "--" + t = toption if thisChar(L) == '-': result.add('-') next(L) + if thisChar(L) == '-': # "---..." + t = tdivider + result.add('-') + while thisChar(L) == '-': + next(L) of Letters: # word t = tword while thisChar(L) in Letters: @@ -112,14 +120,13 @@ proc intValue(v: int): PValue = PValue(kind: vInt, asInt: v) proc floatValue(v: float): PValue = PValue(kind: vFloat, asFloat: v) proc seqValue(v: seq[PValue]): PValue = PValue(kind: vSeq, asSeq: v) - -const MAX_FILES = 30 type PSpec = ref TSpec TSpec = object defVal: string ptype: string + group: int needsValue, multiple, used: bool var progname, usage: string @@ -134,6 +141,7 @@ proc parseSpec(u: string) = var L: PLexer tok: string + groupCounter: int k = 1 let lines = u.splitLines @@ -173,7 +181,8 @@ proc parseSpec(u: string) = k += 1 tok = L.get if tok != ">": fail("argument must be enclosed in <...>") - + elif tok == "---": # divider + inc(groupCounter) if getnext: tok = L.get if tok == ":": # allowed to have colon after flags tok = L.get @@ -205,8 +214,8 @@ proc parseSpec(u: string) = defValue = "false" if name != nil: - # echo("Param: " & name & " type: " & $ftype & " needsvalue: " & $(ftype != "bool") & " default: " & $defValue & " multiple: " & $multiple) - let spec = PSpec(defVal:defValue, ptype: ftype, needsValue: ftype != "bool",multiple:multiple) + #echo("Param: " & name & " type: " & $ftype & " group: " & $groupCounter & " needsvalue: " & $(ftype != "bool") & " default: " & $defValue & " multiple: " & $multiple) + let spec = PSpec(defVal:defValue, ptype: ftype, group: groupCounter, needsValue: ftype != "bool",multiple:multiple) aliases[alias] = name parm_spec[name] = spec @@ -297,12 +306,19 @@ proc parseArguments*(usage: string, args: seq[string]): Table[string,PValue] = if info.used: if flag == "help" or flag == "version": enableChecks = false - - + + # Check maximum group used + var maxGroup = 0 + for item in flagvalues: + info = get_spec(item[0]) + if maxGroup < info.group: + maxGroup = info.group + # any flags not mentioned? for flag,info in parm_spec: if not info.used: - if info.defVal == "": # no default! + # Is there no default and we have used options in this group? + if info.defVal == "" and info.group <= maxGroup: failures.add("required option or argument missing: " & flag) else: flagvalues.add(@[flag,info.defVal]) -- libgit2 0.22.2