diff --git a/blimp.nim b/blimp.nim index c078dc9..34913ae 100644 --- a/blimp.nim +++ b/blimp.nim @@ -2,25 +2,31 @@ import md5, os, osproc, parseopt2, strutils, parsecfg, streams, lapp, subexes # blimp is a little utility program for handling large files # in git repositories. Its inspired by git-fat and s3annex -# but doesn't rely on S3 for storage, is a single binary without -# need for Python, and has less features than git-fat. So far. +# but doesn't rely on S3 for storage - it uses rsync like git-fat. +# It is a single binary without any dependencies. # # Manual use: # # Use "blimp d mybigfile" to deflate it before commit. # Use "blimp i mybigfile" to inflate it back to original size. # -# When deflated the file only has an md5sum string inside it. +# When deflated the file only has: +# "hash:" + +# ...inside it. # -# The file is copied over into: -# /blimpStore/- +# The file is copied over to a local dir: +# /- # # Configuration is in these locations in order: # # ./.blimp.conf # /.blimp.conf -# ~/blimpstore/.blimp.conf +# ~//.blimp.conf # ~/.blimp.conf +# +# This way you can have settings per directory, per git clone, +# per store and per user. A default blimpstore with a commented .blimp.conf +# is created in ~/blimpstore if you run blimp and no .blimp.conf is found. const versionMajor* = 0 @@ -29,14 +35,15 @@ const versionAsString* = $versionMajor & "." & $versionMinor & "." & $versionPatch var - blimpStore, remoteBlimpStore, uploadCommandFormat, downloadCommandFormat, deleteCommandFormat, rsyncPassword: string = nil + blimpStore, remoteBlimpStore, uploadCommandFormat, downloadCommandFormat, deleteCommandFormat, rsyncPassword, blimpVersion: string = nil homeDir, currentDir, gitRootDir: string - verbose: bool + verbose, stdio: bool + stdinContent: string = nil let defaultConfig = """ [rsync] -# Set your local blimpstore directory. +# Set your local blimpstore directory. You can use %home%, %cwd% and %gitroot% in paths, works cross platform. # Example: # # Place it inside the git clone # blimpstore = "%gitroot%/.git/blimpstore" @@ -49,17 +56,20 @@ let # Set this to your remote rsync location remote = "blimpuser@some-rsync-server.com::blimpstore" -# Set this to your rsync password +# Set this to your rsync password, it will be written out as a password-file called .blimp.pass on every rsync. password = "some-good-rsync-password-for-blimpuser" -# The following three formats should not need editing -# $1 is the blimp filename, $2 is remote location and $3 is -# the local blimpstore directory. -# NOTE: The password-file will be created by blimp on every command. +# The following three formats should not need editing. +# $1 is the blimp filename, $2 is remote location and $3 is the local blimpstore directory set above. +# NOTE: The password-file .blimp.pass will be created by blimp on every command, do not remove that option. upload = "rsync --password-file $3/.blimp.pass -avzP $3/$1 $2/" download = "rsync --password-file $3/.blimp.pass -avzP $2/$1 $3/" # This deletes a single file from destination, that is already deleted in source. Yeah... insane! But it works. delete = "rsync --password-file $3/.blimp.pass -dv --delete --existing --ignore-existing --include '$1' --exclude '*' $3/ $2" + +[blimp] +# Minimal version, otherwise stop +# version = 0.2 """ # Find git root dir or nil @@ -110,6 +120,8 @@ proc parseConfFile(filename: string) = if downloadCommandFormat.isNil: downloadCommandFormat = expandDirs(e.value) of "delete": if deleteCommandFormat.isNil: deleteCommandFormat = expandDirs(e.value) + of "version": + if blimpVersion.isNil: blimpVersion = e.value else: quit("Unknown configuration: " & e.key) of cfgOption: @@ -159,14 +171,27 @@ proc remoteDeleteFile(blimpFilename: string) = # Copy content to blimpStore, no upload yet. proc copyToBlimpStore(filename, blimpFilename: string) = if not existsFile(blimpStore / blimpFilename): - copyFile(filename, blimpStore / blimpFilename) - uploadFile(blimpFilename) + if stdio: + try: + writeFile(blimpStore / blimpFilename, stdinContent) + except: + quit("Failed writing file: " & blimpStore / blimpFilename & " from stdin", 1) + else: + copyFile(filename, blimpStore / blimpFilename) + uploadFile(blimpFilename) # Copy content from blimpStore, and downloading first if needed proc copyFromBlimpStore(blimpFilename, filename: string) = if not existsFile(blimpStore / blimpFilename): downloadFile(blimpFilename) - copyFile(blimpStore / blimpFilename, filename) + if stdio: + try: + var content = readFile(blimpStore / blimpFilename) + write(stdout, content) + except: + quit("Failed reading file: " & blimpStore / blimpFilename & " to stdout", 1) + else: + copyFile(blimpStore / blimpFilename, filename) # Delete from blimpStore and remote. proc deleteFromBlimpStore(blimpFilename, filename: string) = @@ -174,17 +199,23 @@ proc deleteFromBlimpStore(blimpFilename, filename: string) = removeFile(blimpStore / blimpFilename) remoteDeleteFile(blimpFilename) -# Pick out blimpFilename (filename & "-" & hash) -proc blimpFileName(filename: string): string= - var hashfile: File - if not open(hashfile, filename): - quit("Failed opening file: " & filename, 4) - let hashline = split(string(readLine(hashfile)), {':'}) +proc blimpFileNameFromString(line: string): string = + let hashline = split(strip(line), {':'}) if hashline[0] == "hash": result = hashline[1] else: result = nil +# Pick out blimpFilename (filename & "-" & hash) +proc blimpFileName(filename: string): string = + if stdio: + blimpFileNameFromString(stdinContent) + else: + var hashfile: File + if not open(hashfile, filename): + quit("Failed opening file: " & filename, 4) + blimpFileNameFromString(string(readLine(hashfile))) + # Get hash and compute blimpFilename proc computeBlimpFilename(filename: string): string = var content: string @@ -204,7 +235,10 @@ proc deflate(filename: string) = else: blimpFilename = computeBlimpFilename(filename) copyToBlimpStore(filename, blimpFilename) - writeFile(filename, "hash:" & blimpFilename) + if stdio: + write(stdout, "hash:" & blimpFilename) + else: + writeFile(filename, "hash:" & blimpFilename) if verbose: echo("\t" & filename & " deflated.") # Parse out hash from hash stub and copy back original content from blimpStore. @@ -253,34 +287,69 @@ proc dumpConfig() = echo "\tdownloadCommandFormat: " & downloadCommandFormat echo "\tdeleteCommandFormat: " & deleteCommandFormat echo "\trsyncPassword: " & $rsyncPassword + echo "\tblimpVersion: " & $blimpVersion echo "\n" let help = """ blimp [options] -h,--help Show this --version Show version of blimp - -v,--verbose Verbosity + -v,--verbose Verbosity, only works without -s + -s,--stdio If given, use stdin/stdout for content. (string) (d)eflate, (i)nflate, remove - (string...) One or more filepaths to inflate/deflate - - Edit ~/blimpstore/.blimp.conf or /.blimp.conf and set a proper - remote and the proper rsync password to use. - - Deflate is run before you add the big file to the index for committing. - Deflate will replace the file contents with a hash, and copy the - real content to your local blimpstore, and if configured also upload it to - remote, using rsync. - - Inflate will bring back the original content by copying from - your local blimpstore, and if its not there, first downloading from the remote. - Use this whenever you need to work/edit the big file - in order to get - its real content. - - Remove (no single character shortcut) will remove the file(s) content - both from the local blimpstore and from the remote. This only removes - the current content version, not older versions. The file itself is first - inflated, if needed, and not deleted. This only "unblimps" the file. - """ + (string...) One or more filepaths to inflate/deflate + + blimp is a little utility program for handling large files + in git repositories. Its inspired by git-fat and s3annex + but doesn't rely on S3 for storage - it uses rsync like git-fat. + It is a single binary without any dependencies. + + Manual use: + + Use "blimp d mybigfile" to deflate it before commit. + Use "blimp i mybigfile" to inflate it back to original size. + + When deflated the file only has: + "hash:" "-" + ...inside it. + + Deflate is run before you add the big file to the index for committing. + Deflate will replace the file contents with a hash, and copy the + real content to your local blimpstore: + + "blimpstore"/- + + ...and if configured also upload it to "remote", using rsync. + + Configuration is in these locations in order: + + ./.blimp.conf + "gitroot"/.blimp.conf + ~//.blimp.conf + ~/.blimp.conf + + This way you can have settings per directory, per git clone, + per store and per user. A default blimpstore with a commented .blimp.conf + is created in ~/blimpstore if you run blimp and no .blimp.conf is found. + + Edit ~/blimpstore/.blimp.conf (or in another location) and set a proper + remote and the proper rsync password. This ensures that its also properly + synced with a master rsync repository that is typically shared. + + Inflate will bring back the original content by copying from + your local blimpstore, and if its not there, first downloading from the remote. + Use this whenever you need to work/edit the big file - in order to get + its real content. + + The filenames given are all processed. If -s is used content is processed via + stdin/stdout and only one filename can be passed. This is used when running blimp + via a git filter (smudge/clean). + + The remove command (no single character shortcut) will remove the file(s) content + both from the local blimpstore and from the remote. This only removes + the current content version, not older versions. The file itself is first + inflated, if needed, and not deleted. This only "unblimps" the file. +""" ################################ main ##################################### # Set some dirs @@ -292,6 +361,18 @@ gitRootDir = gitRoot() # Using lapp to get args, on parsing failure this will show usage automatically var args = parse(help) verbose = args["verbose"].asBool +stdio = args["stdio"].asBool + +# Can't do verbose with -s, that messes up stdout, +# read in all of stdin once and for all +if stdio: + verbose = false + try: + stdinContent = readAll(stdin) + close(stdin) + except: + quit("Failed reading stdin", 1) + # Parse configuration files, may shadow and override each other parseConfFile(currentDir / ".blimp.conf") @@ -312,6 +393,10 @@ if verbose: dumpConfig() if args.showHelp: quit(help) if args.showVersion: quit("blimp version: " & versionAsString) +# Check blimpVersion +if not blimpVersion.isNil and blimpVersion != versionAsString: + quit("Wrong version of blimp, configuration wants: " & blimpVersion) + let command = args["command"].asString let filenames = args["filenames"].asSeq diff --git a/blimp.nimble b/blimp.nimble index 9c707e7..72b6cab 100644 --- a/blimp.nimble +++ b/blimp.nimble @@ -1,6 +1,6 @@ [Package] name = "blimp" -version = "0.2" +version = "0.3" author = "Göran Krampe" description = "Utility that helps with big files in git, very similar to git-fat, s3annnex etc." license = "MIT"