Commit aaf9bcef10ab91e7d45c444f5ba728c4d77751f2

Authored by Göran Krampe
1 parent 78569137

Upped to version 0.3, stdio supported so it can work as a git filter.

Showing 2 changed files with 131 additions and 46 deletions
blimp.nim
@@ -2,25 +2,31 @@ import md5, os, osproc, parseopt2, strutils, parsecfg, streams, lapp, subexes @@ -2,25 +2,31 @@ import md5, os, osproc, parseopt2, strutils, parsecfg, streams, lapp, subexes
2 2
3 # blimp is a little utility program for handling large files 3 # blimp is a little utility program for handling large files
4 # in git repositories. Its inspired by git-fat and s3annex 4 # in git repositories. Its inspired by git-fat and s3annex
5 -# but doesn't rely on S3 for storage, is a single binary without  
6 -# need for Python, and has less features than git-fat. So far. 5 +# but doesn't rely on S3 for storage - it uses rsync like git-fat.
  6 +# It is a single binary without any dependencies.
7 # 7 #
8 # Manual use: 8 # Manual use:
9 # 9 #
10 # Use "blimp d mybigfile" to deflate it before commit. 10 # Use "blimp d mybigfile" to deflate it before commit.
11 # Use "blimp i mybigfile" to inflate it back to original size. 11 # Use "blimp i mybigfile" to inflate it back to original size.
12 # 12 #
13 -# When deflated the file only has an md5sum string inside it. 13 +# When deflated the file only has:
  14 +# "hash:" <filename> + <an md5sum>
  15 +# ...inside it.
14 # 16 #
15 -# The file is copied over into:  
16 -# <homedir>/blimpStore/<originalfilename>-<md5sum> 17 +# The file is copied over to a local dir:
  18 +# <blimpstore>/<filename>-<md5sum>
17 # 19 #
18 # Configuration is in these locations in order: 20 # Configuration is in these locations in order:
19 # 21 #
20 # ./.blimp.conf 22 # ./.blimp.conf
21 # <gitroot>/.blimp.conf 23 # <gitroot>/.blimp.conf
22 -# ~/blimpstore/.blimp.conf 24 +# ~/<blimpstore>/.blimp.conf
23 # ~/.blimp.conf 25 # ~/.blimp.conf
  26 +#
  27 +# This way you can have settings per directory, per git clone,
  28 +# per store and per user. A default blimpstore with a commented .blimp.conf
  29 +# is created in ~/blimpstore if you run blimp and no .blimp.conf is found.
24 30
25 const 31 const
26 versionMajor* = 0 32 versionMajor* = 0
@@ -29,14 +35,15 @@ const @@ -29,14 +35,15 @@ const
29 versionAsString* = $versionMajor & "." & $versionMinor & "." & $versionPatch 35 versionAsString* = $versionMajor & "." & $versionMinor & "." & $versionPatch
30 36
31 var 37 var
32 - blimpStore, remoteBlimpStore, uploadCommandFormat, downloadCommandFormat, deleteCommandFormat, rsyncPassword: string = nil 38 + blimpStore, remoteBlimpStore, uploadCommandFormat, downloadCommandFormat, deleteCommandFormat, rsyncPassword, blimpVersion: string = nil
33 homeDir, currentDir, gitRootDir: string 39 homeDir, currentDir, gitRootDir: string
34 - verbose: bool 40 + verbose, stdio: bool
  41 + stdinContent: string = nil
35 42
36 let 43 let
37 defaultConfig = """ 44 defaultConfig = """
38 [rsync] 45 [rsync]
39 -# Set your local blimpstore directory. 46 +# Set your local blimpstore directory. You can use %home%, %cwd% and %gitroot% in paths, works cross platform.
40 # Example: 47 # Example:
41 # # Place it inside the git clone 48 # # Place it inside the git clone
42 # blimpstore = "%gitroot%/.git/blimpstore" 49 # blimpstore = "%gitroot%/.git/blimpstore"
@@ -49,17 +56,20 @@ let @@ -49,17 +56,20 @@ let
49 56
50 # Set this to your remote rsync location 57 # Set this to your remote rsync location
51 remote = "blimpuser@some-rsync-server.com::blimpstore" 58 remote = "blimpuser@some-rsync-server.com::blimpstore"
52 -# Set this to your rsync password 59 +# Set this to your rsync password, it will be written out as a password-file called .blimp.pass on every rsync.
53 password = "some-good-rsync-password-for-blimpuser" 60 password = "some-good-rsync-password-for-blimpuser"
54 61
55 -# The following three formats should not need editing  
56 -# $1 is the blimp filename, $2 is remote location and $3 is  
57 -# the local blimpstore directory.  
58 -# NOTE: The password-file will be created by blimp on every command. 62 +# The following three formats should not need editing.
  63 +# $1 is the blimp filename, $2 is remote location and $3 is the local blimpstore directory set above.
  64 +# NOTE: The password-file .blimp.pass will be created by blimp on every command, do not remove that option.
59 upload = "rsync --password-file $3/.blimp.pass -avzP $3/$1 $2/" 65 upload = "rsync --password-file $3/.blimp.pass -avzP $3/$1 $2/"
60 download = "rsync --password-file $3/.blimp.pass -avzP $2/$1 $3/" 66 download = "rsync --password-file $3/.blimp.pass -avzP $2/$1 $3/"
61 # This deletes a single file from destination, that is already deleted in source. Yeah... insane! But it works. 67 # This deletes a single file from destination, that is already deleted in source. Yeah... insane! But it works.
62 delete = "rsync --password-file $3/.blimp.pass -dv --delete --existing --ignore-existing --include '$1' --exclude '*' $3/ $2" 68 delete = "rsync --password-file $3/.blimp.pass -dv --delete --existing --ignore-existing --include '$1' --exclude '*' $3/ $2"
  69 +
  70 +[blimp]
  71 +# Minimal version, otherwise stop
  72 +# version = 0.2
63 """ 73 """
64 74
65 # Find git root dir or nil 75 # Find git root dir or nil
@@ -110,6 +120,8 @@ proc parseConfFile(filename: string) = @@ -110,6 +120,8 @@ proc parseConfFile(filename: string) =
110 if downloadCommandFormat.isNil: downloadCommandFormat = expandDirs(e.value) 120 if downloadCommandFormat.isNil: downloadCommandFormat = expandDirs(e.value)
111 of "delete": 121 of "delete":
112 if deleteCommandFormat.isNil: deleteCommandFormat = expandDirs(e.value) 122 if deleteCommandFormat.isNil: deleteCommandFormat = expandDirs(e.value)
  123 + of "version":
  124 + if blimpVersion.isNil: blimpVersion = e.value
113 else: 125 else:
114 quit("Unknown configuration: " & e.key) 126 quit("Unknown configuration: " & e.key)
115 of cfgOption: 127 of cfgOption:
@@ -159,14 +171,27 @@ proc remoteDeleteFile(blimpFilename: string) = @@ -159,14 +171,27 @@ proc remoteDeleteFile(blimpFilename: string) =
159 # Copy content to blimpStore, no upload yet. 171 # Copy content to blimpStore, no upload yet.
160 proc copyToBlimpStore(filename, blimpFilename: string) = 172 proc copyToBlimpStore(filename, blimpFilename: string) =
161 if not existsFile(blimpStore / blimpFilename): 173 if not existsFile(blimpStore / blimpFilename):
162 - copyFile(filename, blimpStore / blimpFilename)  
163 - uploadFile(blimpFilename) 174 + if stdio:
  175 + try:
  176 + writeFile(blimpStore / blimpFilename, stdinContent)
  177 + except:
  178 + quit("Failed writing file: " & blimpStore / blimpFilename & " from stdin", 1)
  179 + else:
  180 + copyFile(filename, blimpStore / blimpFilename)
  181 + uploadFile(blimpFilename)
164 182
165 # Copy content from blimpStore, and downloading first if needed 183 # Copy content from blimpStore, and downloading first if needed
166 proc copyFromBlimpStore(blimpFilename, filename: string) = 184 proc copyFromBlimpStore(blimpFilename, filename: string) =
167 if not existsFile(blimpStore / blimpFilename): 185 if not existsFile(blimpStore / blimpFilename):
168 downloadFile(blimpFilename) 186 downloadFile(blimpFilename)
169 - copyFile(blimpStore / blimpFilename, filename) 187 + if stdio:
  188 + try:
  189 + var content = readFile(blimpStore / blimpFilename)
  190 + write(stdout, content)
  191 + except:
  192 + quit("Failed reading file: " & blimpStore / blimpFilename & " to stdout", 1)
  193 + else:
  194 + copyFile(blimpStore / blimpFilename, filename)
170 195
171 # Delete from blimpStore and remote. 196 # Delete from blimpStore and remote.
172 proc deleteFromBlimpStore(blimpFilename, filename: string) = 197 proc deleteFromBlimpStore(blimpFilename, filename: string) =
@@ -174,17 +199,23 @@ proc deleteFromBlimpStore(blimpFilename, filename: string) = @@ -174,17 +199,23 @@ proc deleteFromBlimpStore(blimpFilename, filename: string) =
174 removeFile(blimpStore / blimpFilename) 199 removeFile(blimpStore / blimpFilename)
175 remoteDeleteFile(blimpFilename) 200 remoteDeleteFile(blimpFilename)
176 201
177 -# Pick out blimpFilename (filename & "-" & hash)  
178 -proc blimpFileName(filename: string): string=  
179 - var hashfile: File  
180 - if not open(hashfile, filename):  
181 - quit("Failed opening file: " & filename, 4)  
182 - let hashline = split(string(readLine(hashfile)), {':'}) 202 +proc blimpFileNameFromString(line: string): string =
  203 + let hashline = split(strip(line), {':'})
183 if hashline[0] == "hash": 204 if hashline[0] == "hash":
184 result = hashline[1] 205 result = hashline[1]
185 else: 206 else:
186 result = nil 207 result = nil
187 208
  209 +# Pick out blimpFilename (filename & "-" & hash)
  210 +proc blimpFileName(filename: string): string =
  211 + if stdio:
  212 + blimpFileNameFromString(stdinContent)
  213 + else:
  214 + var hashfile: File
  215 + if not open(hashfile, filename):
  216 + quit("Failed opening file: " & filename, 4)
  217 + blimpFileNameFromString(string(readLine(hashfile)))
  218 +
188 # Get hash and compute blimpFilename 219 # Get hash and compute blimpFilename
189 proc computeBlimpFilename(filename: string): string = 220 proc computeBlimpFilename(filename: string): string =
190 var content: string 221 var content: string
@@ -204,7 +235,10 @@ proc deflate(filename: string) = @@ -204,7 +235,10 @@ proc deflate(filename: string) =
204 else: 235 else:
205 blimpFilename = computeBlimpFilename(filename) 236 blimpFilename = computeBlimpFilename(filename)
206 copyToBlimpStore(filename, blimpFilename) 237 copyToBlimpStore(filename, blimpFilename)
207 - writeFile(filename, "hash:" & blimpFilename) 238 + if stdio:
  239 + write(stdout, "hash:" & blimpFilename)
  240 + else:
  241 + writeFile(filename, "hash:" & blimpFilename)
208 if verbose: echo("\t" & filename & " deflated.") 242 if verbose: echo("\t" & filename & " deflated.")
209 243
210 # Parse out hash from hash stub and copy back original content from blimpStore. 244 # Parse out hash from hash stub and copy back original content from blimpStore.
@@ -253,34 +287,69 @@ proc dumpConfig() = @@ -253,34 +287,69 @@ proc dumpConfig() =
253 echo "\tdownloadCommandFormat: " & downloadCommandFormat 287 echo "\tdownloadCommandFormat: " & downloadCommandFormat
254 echo "\tdeleteCommandFormat: " & deleteCommandFormat 288 echo "\tdeleteCommandFormat: " & deleteCommandFormat
255 echo "\trsyncPassword: " & $rsyncPassword 289 echo "\trsyncPassword: " & $rsyncPassword
  290 + echo "\tblimpVersion: " & $blimpVersion
256 echo "\n" 291 echo "\n"
257 292
258 let help = """ 293 let help = """
259 blimp [options] <command> <filenames...> 294 blimp [options] <command> <filenames...>
260 -h,--help Show this 295 -h,--help Show this
261 --version Show version of blimp 296 --version Show version of blimp
262 - -v,--verbose Verbosity 297 + -v,--verbose Verbosity, only works without -s
  298 + -s,--stdio If given, use stdin/stdout for content.
263 <command> (string) (d)eflate, (i)nflate, remove 299 <command> (string) (d)eflate, (i)nflate, remove
264 - <filenames> (string...) One or more filepaths to inflate/deflate  
265 -  
266 - Edit ~/blimpstore/.blimp.conf or <gitroot>/.blimp.conf and set a proper  
267 - remote and the proper rsync password to use.  
268 -  
269 - Deflate is run before you add the big file to the index for committing.  
270 - Deflate will replace the file contents with a hash, and copy the  
271 - real content to your local blimpstore, and if configured also upload it to  
272 - remote, using rsync.  
273 -  
274 - Inflate will bring back the original content by copying from  
275 - your local blimpstore, and if its not there, first downloading from the remote.  
276 - Use this whenever you need to work/edit the big file - in order to get  
277 - its real content.  
278 -  
279 - Remove (no single character shortcut) will remove the file(s) content  
280 - both from the local blimpstore and from the remote. This only removes  
281 - the current content version, not older versions. The file itself is first  
282 - inflated, if needed, and not deleted. This only "unblimps" the file.  
283 - """ 300 + <filenames> (string...) One or more filepaths to inflate/deflate
  301 +
  302 + blimp is a little utility program for handling large files
  303 + in git repositories. Its inspired by git-fat and s3annex
  304 + but doesn't rely on S3 for storage - it uses rsync like git-fat.
  305 + It is a single binary without any dependencies.
  306 +
  307 + Manual use:
  308 +
  309 + Use "blimp d mybigfile" to deflate it before commit.
  310 + Use "blimp i mybigfile" to inflate it back to original size.
  311 +
  312 + When deflated the file only has:
  313 + "hash:" <filename> "-" <md5sum>
  314 + ...inside it.
  315 +
  316 + Deflate is run before you add the big file to the index for committing.
  317 + Deflate will replace the file contents with a hash, and copy the
  318 + real content to your local blimpstore:
  319 +
  320 + "blimpstore"/<filename>-<md5sum>
  321 +
  322 + ...and if configured also upload it to "remote", using rsync.
  323 +
  324 + Configuration is in these locations in order:
  325 +
  326 + ./.blimp.conf
  327 + "gitroot"/.blimp.conf
  328 + ~/<blimpstore>/.blimp.conf
  329 + ~/.blimp.conf
  330 +
  331 + This way you can have settings per directory, per git clone,
  332 + per store and per user. A default blimpstore with a commented .blimp.conf
  333 + is created in ~/blimpstore if you run blimp and no .blimp.conf is found.
  334 +
  335 + Edit ~/blimpstore/.blimp.conf (or in another location) and set a proper
  336 + remote and the proper rsync password. This ensures that its also properly
  337 + synced with a master rsync repository that is typically shared.
  338 +
  339 + Inflate will bring back the original content by copying from
  340 + your local blimpstore, and if its not there, first downloading from the remote.
  341 + Use this whenever you need to work/edit the big file - in order to get
  342 + its real content.
  343 +
  344 + The filenames given are all processed. If -s is used content is processed via
  345 + stdin/stdout and only one filename can be passed. This is used when running blimp
  346 + via a git filter (smudge/clean).
  347 +
  348 + The remove command (no single character shortcut) will remove the file(s) content
  349 + both from the local blimpstore and from the remote. This only removes
  350 + the current content version, not older versions. The file itself is first
  351 + inflated, if needed, and not deleted. This only "unblimps" the file.
  352 +"""
284 353
285 ################################ main ##################################### 354 ################################ main #####################################
286 # Set some dirs 355 # Set some dirs
@@ -292,6 +361,18 @@ gitRootDir = gitRoot() @@ -292,6 +361,18 @@ gitRootDir = gitRoot()
292 # Using lapp to get args, on parsing failure this will show usage automatically 361 # Using lapp to get args, on parsing failure this will show usage automatically
293 var args = parse(help) 362 var args = parse(help)
294 verbose = args["verbose"].asBool 363 verbose = args["verbose"].asBool
  364 +stdio = args["stdio"].asBool
  365 +
  366 +# Can't do verbose with -s, that messes up stdout,
  367 +# read in all of stdin once and for all
  368 +if stdio:
  369 + verbose = false
  370 + try:
  371 + stdinContent = readAll(stdin)
  372 + close(stdin)
  373 + except:
  374 + quit("Failed reading stdin", 1)
  375 +
295 376
296 # Parse configuration files, may shadow and override each other 377 # Parse configuration files, may shadow and override each other
297 parseConfFile(currentDir / ".blimp.conf") 378 parseConfFile(currentDir / ".blimp.conf")
@@ -312,6 +393,10 @@ if verbose: dumpConfig() @@ -312,6 +393,10 @@ if verbose: dumpConfig()
312 if args.showHelp: quit(help) 393 if args.showHelp: quit(help)
313 if args.showVersion: quit("blimp version: " & versionAsString) 394 if args.showVersion: quit("blimp version: " & versionAsString)
314 395
  396 +# Check blimpVersion
  397 +if not blimpVersion.isNil and blimpVersion != versionAsString:
  398 + quit("Wrong version of blimp, configuration wants: " & blimpVersion)
  399 +
315 let command = args["command"].asString 400 let command = args["command"].asString
316 let filenames = args["filenames"].asSeq 401 let filenames = args["filenames"].asSeq
317 402
blimp.nimble
1 [Package] 1 [Package]
2 name = "blimp" 2 name = "blimp"
3 -version = "0.2" 3 +version = "0.3"
4 author = "Göran Krampe" 4 author = "Göran Krampe"
5 description = "Utility that helps with big files in git, very similar to git-fat, s3annnex etc." 5 description = "Utility that helps with big files in git, very similar to git-fat, s3annnex etc."
6 license = "MIT" 6 license = "MIT"