Commit aaf9bcef10ab91e7d45c444f5ba728c4d77751f2
1 parent
78569137
Upped to version 0.3, stdio supported so it can work as a git filter.
Showing
2 changed files
with
131 additions
and
46 deletions
blimp.nim
@@ -2,25 +2,31 @@ import md5, os, osproc, parseopt2, strutils, parsecfg, streams, lapp, subexes | @@ -2,25 +2,31 @@ import md5, os, osproc, parseopt2, strutils, parsecfg, streams, lapp, subexes | ||
2 | 2 | ||
3 | # blimp is a little utility program for handling large files | 3 | # blimp is a little utility program for handling large files |
4 | # in git repositories. Its inspired by git-fat and s3annex | 4 | # in git repositories. Its inspired by git-fat and s3annex |
5 | -# but doesn't rely on S3 for storage, is a single binary without | ||
6 | -# need for Python, and has less features than git-fat. So far. | 5 | +# but doesn't rely on S3 for storage - it uses rsync like git-fat. |
6 | +# It is a single binary without any dependencies. | ||
7 | # | 7 | # |
8 | # Manual use: | 8 | # Manual use: |
9 | # | 9 | # |
10 | # Use "blimp d mybigfile" to deflate it before commit. | 10 | # Use "blimp d mybigfile" to deflate it before commit. |
11 | # Use "blimp i mybigfile" to inflate it back to original size. | 11 | # Use "blimp i mybigfile" to inflate it back to original size. |
12 | # | 12 | # |
13 | -# When deflated the file only has an md5sum string inside it. | 13 | +# When deflated the file only has: |
14 | +# "hash:" <filename> + <an md5sum> | ||
15 | +# ...inside it. | ||
14 | # | 16 | # |
15 | -# The file is copied over into: | ||
16 | -# <homedir>/blimpStore/<originalfilename>-<md5sum> | 17 | +# The file is copied over to a local dir: |
18 | +# <blimpstore>/<filename>-<md5sum> | ||
17 | # | 19 | # |
18 | # Configuration is in these locations in order: | 20 | # Configuration is in these locations in order: |
19 | # | 21 | # |
20 | # ./.blimp.conf | 22 | # ./.blimp.conf |
21 | # <gitroot>/.blimp.conf | 23 | # <gitroot>/.blimp.conf |
22 | -# ~/blimpstore/.blimp.conf | 24 | +# ~/<blimpstore>/.blimp.conf |
23 | # ~/.blimp.conf | 25 | # ~/.blimp.conf |
26 | +# | ||
27 | +# This way you can have settings per directory, per git clone, | ||
28 | +# per store and per user. A default blimpstore with a commented .blimp.conf | ||
29 | +# is created in ~/blimpstore if you run blimp and no .blimp.conf is found. | ||
24 | 30 | ||
25 | const | 31 | const |
26 | versionMajor* = 0 | 32 | versionMajor* = 0 |
@@ -29,14 +35,15 @@ const | @@ -29,14 +35,15 @@ const | ||
29 | versionAsString* = $versionMajor & "." & $versionMinor & "." & $versionPatch | 35 | versionAsString* = $versionMajor & "." & $versionMinor & "." & $versionPatch |
30 | 36 | ||
31 | var | 37 | var |
32 | - blimpStore, remoteBlimpStore, uploadCommandFormat, downloadCommandFormat, deleteCommandFormat, rsyncPassword: string = nil | 38 | + blimpStore, remoteBlimpStore, uploadCommandFormat, downloadCommandFormat, deleteCommandFormat, rsyncPassword, blimpVersion: string = nil |
33 | homeDir, currentDir, gitRootDir: string | 39 | homeDir, currentDir, gitRootDir: string |
34 | - verbose: bool | 40 | + verbose, stdio: bool |
41 | + stdinContent: string = nil | ||
35 | 42 | ||
36 | let | 43 | let |
37 | defaultConfig = """ | 44 | defaultConfig = """ |
38 | [rsync] | 45 | [rsync] |
39 | -# Set your local blimpstore directory. | 46 | +# Set your local blimpstore directory. You can use %home%, %cwd% and %gitroot% in paths, works cross platform. |
40 | # Example: | 47 | # Example: |
41 | # # Place it inside the git clone | 48 | # # Place it inside the git clone |
42 | # blimpstore = "%gitroot%/.git/blimpstore" | 49 | # blimpstore = "%gitroot%/.git/blimpstore" |
@@ -49,17 +56,20 @@ let | @@ -49,17 +56,20 @@ let | ||
49 | 56 | ||
50 | # Set this to your remote rsync location | 57 | # Set this to your remote rsync location |
51 | remote = "blimpuser@some-rsync-server.com::blimpstore" | 58 | remote = "blimpuser@some-rsync-server.com::blimpstore" |
52 | -# Set this to your rsync password | 59 | +# Set this to your rsync password, it will be written out as a password-file called .blimp.pass on every rsync. |
53 | password = "some-good-rsync-password-for-blimpuser" | 60 | password = "some-good-rsync-password-for-blimpuser" |
54 | 61 | ||
55 | -# The following three formats should not need editing | ||
56 | -# $1 is the blimp filename, $2 is remote location and $3 is | ||
57 | -# the local blimpstore directory. | ||
58 | -# NOTE: The password-file will be created by blimp on every command. | 62 | +# The following three formats should not need editing. |
63 | +# $1 is the blimp filename, $2 is remote location and $3 is the local blimpstore directory set above. | ||
64 | +# NOTE: The password-file .blimp.pass will be created by blimp on every command, do not remove that option. | ||
59 | upload = "rsync --password-file $3/.blimp.pass -avzP $3/$1 $2/" | 65 | upload = "rsync --password-file $3/.blimp.pass -avzP $3/$1 $2/" |
60 | download = "rsync --password-file $3/.blimp.pass -avzP $2/$1 $3/" | 66 | download = "rsync --password-file $3/.blimp.pass -avzP $2/$1 $3/" |
61 | # This deletes a single file from destination, that is already deleted in source. Yeah... insane! But it works. | 67 | # This deletes a single file from destination, that is already deleted in source. Yeah... insane! But it works. |
62 | delete = "rsync --password-file $3/.blimp.pass -dv --delete --existing --ignore-existing --include '$1' --exclude '*' $3/ $2" | 68 | delete = "rsync --password-file $3/.blimp.pass -dv --delete --existing --ignore-existing --include '$1' --exclude '*' $3/ $2" |
69 | + | ||
70 | +[blimp] | ||
71 | +# Minimal version, otherwise stop | ||
72 | +# version = 0.2 | ||
63 | """ | 73 | """ |
64 | 74 | ||
65 | # Find git root dir or nil | 75 | # Find git root dir or nil |
@@ -110,6 +120,8 @@ proc parseConfFile(filename: string) = | @@ -110,6 +120,8 @@ proc parseConfFile(filename: string) = | ||
110 | if downloadCommandFormat.isNil: downloadCommandFormat = expandDirs(e.value) | 120 | if downloadCommandFormat.isNil: downloadCommandFormat = expandDirs(e.value) |
111 | of "delete": | 121 | of "delete": |
112 | if deleteCommandFormat.isNil: deleteCommandFormat = expandDirs(e.value) | 122 | if deleteCommandFormat.isNil: deleteCommandFormat = expandDirs(e.value) |
123 | + of "version": | ||
124 | + if blimpVersion.isNil: blimpVersion = e.value | ||
113 | else: | 125 | else: |
114 | quit("Unknown configuration: " & e.key) | 126 | quit("Unknown configuration: " & e.key) |
115 | of cfgOption: | 127 | of cfgOption: |
@@ -159,14 +171,27 @@ proc remoteDeleteFile(blimpFilename: string) = | @@ -159,14 +171,27 @@ proc remoteDeleteFile(blimpFilename: string) = | ||
159 | # Copy content to blimpStore, no upload yet. | 171 | # Copy content to blimpStore, no upload yet. |
160 | proc copyToBlimpStore(filename, blimpFilename: string) = | 172 | proc copyToBlimpStore(filename, blimpFilename: string) = |
161 | if not existsFile(blimpStore / blimpFilename): | 173 | if not existsFile(blimpStore / blimpFilename): |
162 | - copyFile(filename, blimpStore / blimpFilename) | ||
163 | - uploadFile(blimpFilename) | 174 | + if stdio: |
175 | + try: | ||
176 | + writeFile(blimpStore / blimpFilename, stdinContent) | ||
177 | + except: | ||
178 | + quit("Failed writing file: " & blimpStore / blimpFilename & " from stdin", 1) | ||
179 | + else: | ||
180 | + copyFile(filename, blimpStore / blimpFilename) | ||
181 | + uploadFile(blimpFilename) | ||
164 | 182 | ||
165 | # Copy content from blimpStore, and downloading first if needed | 183 | # Copy content from blimpStore, and downloading first if needed |
166 | proc copyFromBlimpStore(blimpFilename, filename: string) = | 184 | proc copyFromBlimpStore(blimpFilename, filename: string) = |
167 | if not existsFile(blimpStore / blimpFilename): | 185 | if not existsFile(blimpStore / blimpFilename): |
168 | downloadFile(blimpFilename) | 186 | downloadFile(blimpFilename) |
169 | - copyFile(blimpStore / blimpFilename, filename) | 187 | + if stdio: |
188 | + try: | ||
189 | + var content = readFile(blimpStore / blimpFilename) | ||
190 | + write(stdout, content) | ||
191 | + except: | ||
192 | + quit("Failed reading file: " & blimpStore / blimpFilename & " to stdout", 1) | ||
193 | + else: | ||
194 | + copyFile(blimpStore / blimpFilename, filename) | ||
170 | 195 | ||
171 | # Delete from blimpStore and remote. | 196 | # Delete from blimpStore and remote. |
172 | proc deleteFromBlimpStore(blimpFilename, filename: string) = | 197 | proc deleteFromBlimpStore(blimpFilename, filename: string) = |
@@ -174,17 +199,23 @@ proc deleteFromBlimpStore(blimpFilename, filename: string) = | @@ -174,17 +199,23 @@ proc deleteFromBlimpStore(blimpFilename, filename: string) = | ||
174 | removeFile(blimpStore / blimpFilename) | 199 | removeFile(blimpStore / blimpFilename) |
175 | remoteDeleteFile(blimpFilename) | 200 | remoteDeleteFile(blimpFilename) |
176 | 201 | ||
177 | -# Pick out blimpFilename (filename & "-" & hash) | ||
178 | -proc blimpFileName(filename: string): string= | ||
179 | - var hashfile: File | ||
180 | - if not open(hashfile, filename): | ||
181 | - quit("Failed opening file: " & filename, 4) | ||
182 | - let hashline = split(string(readLine(hashfile)), {':'}) | 202 | +proc blimpFileNameFromString(line: string): string = |
203 | + let hashline = split(strip(line), {':'}) | ||
183 | if hashline[0] == "hash": | 204 | if hashline[0] == "hash": |
184 | result = hashline[1] | 205 | result = hashline[1] |
185 | else: | 206 | else: |
186 | result = nil | 207 | result = nil |
187 | 208 | ||
209 | +# Pick out blimpFilename (filename & "-" & hash) | ||
210 | +proc blimpFileName(filename: string): string = | ||
211 | + if stdio: | ||
212 | + blimpFileNameFromString(stdinContent) | ||
213 | + else: | ||
214 | + var hashfile: File | ||
215 | + if not open(hashfile, filename): | ||
216 | + quit("Failed opening file: " & filename, 4) | ||
217 | + blimpFileNameFromString(string(readLine(hashfile))) | ||
218 | + | ||
188 | # Get hash and compute blimpFilename | 219 | # Get hash and compute blimpFilename |
189 | proc computeBlimpFilename(filename: string): string = | 220 | proc computeBlimpFilename(filename: string): string = |
190 | var content: string | 221 | var content: string |
@@ -204,7 +235,10 @@ proc deflate(filename: string) = | @@ -204,7 +235,10 @@ proc deflate(filename: string) = | ||
204 | else: | 235 | else: |
205 | blimpFilename = computeBlimpFilename(filename) | 236 | blimpFilename = computeBlimpFilename(filename) |
206 | copyToBlimpStore(filename, blimpFilename) | 237 | copyToBlimpStore(filename, blimpFilename) |
207 | - writeFile(filename, "hash:" & blimpFilename) | 238 | + if stdio: |
239 | + write(stdout, "hash:" & blimpFilename) | ||
240 | + else: | ||
241 | + writeFile(filename, "hash:" & blimpFilename) | ||
208 | if verbose: echo("\t" & filename & " deflated.") | 242 | if verbose: echo("\t" & filename & " deflated.") |
209 | 243 | ||
210 | # Parse out hash from hash stub and copy back original content from blimpStore. | 244 | # Parse out hash from hash stub and copy back original content from blimpStore. |
@@ -253,34 +287,69 @@ proc dumpConfig() = | @@ -253,34 +287,69 @@ proc dumpConfig() = | ||
253 | echo "\tdownloadCommandFormat: " & downloadCommandFormat | 287 | echo "\tdownloadCommandFormat: " & downloadCommandFormat |
254 | echo "\tdeleteCommandFormat: " & deleteCommandFormat | 288 | echo "\tdeleteCommandFormat: " & deleteCommandFormat |
255 | echo "\trsyncPassword: " & $rsyncPassword | 289 | echo "\trsyncPassword: " & $rsyncPassword |
290 | + echo "\tblimpVersion: " & $blimpVersion | ||
256 | echo "\n" | 291 | echo "\n" |
257 | 292 | ||
258 | let help = """ | 293 | let help = """ |
259 | blimp [options] <command> <filenames...> | 294 | blimp [options] <command> <filenames...> |
260 | -h,--help Show this | 295 | -h,--help Show this |
261 | --version Show version of blimp | 296 | --version Show version of blimp |
262 | - -v,--verbose Verbosity | 297 | + -v,--verbose Verbosity, only works without -s |
298 | + -s,--stdio If given, use stdin/stdout for content. | ||
263 | <command> (string) (d)eflate, (i)nflate, remove | 299 | <command> (string) (d)eflate, (i)nflate, remove |
264 | - <filenames> (string...) One or more filepaths to inflate/deflate | ||
265 | - | ||
266 | - Edit ~/blimpstore/.blimp.conf or <gitroot>/.blimp.conf and set a proper | ||
267 | - remote and the proper rsync password to use. | ||
268 | - | ||
269 | - Deflate is run before you add the big file to the index for committing. | ||
270 | - Deflate will replace the file contents with a hash, and copy the | ||
271 | - real content to your local blimpstore, and if configured also upload it to | ||
272 | - remote, using rsync. | ||
273 | - | ||
274 | - Inflate will bring back the original content by copying from | ||
275 | - your local blimpstore, and if its not there, first downloading from the remote. | ||
276 | - Use this whenever you need to work/edit the big file - in order to get | ||
277 | - its real content. | ||
278 | - | ||
279 | - Remove (no single character shortcut) will remove the file(s) content | ||
280 | - both from the local blimpstore and from the remote. This only removes | ||
281 | - the current content version, not older versions. The file itself is first | ||
282 | - inflated, if needed, and not deleted. This only "unblimps" the file. | ||
283 | - """ | 300 | + <filenames> (string...) One or more filepaths to inflate/deflate |
301 | + | ||
302 | + blimp is a little utility program for handling large files | ||
303 | + in git repositories. Its inspired by git-fat and s3annex | ||
304 | + but doesn't rely on S3 for storage - it uses rsync like git-fat. | ||
305 | + It is a single binary without any dependencies. | ||
306 | + | ||
307 | + Manual use: | ||
308 | + | ||
309 | + Use "blimp d mybigfile" to deflate it before commit. | ||
310 | + Use "blimp i mybigfile" to inflate it back to original size. | ||
311 | + | ||
312 | + When deflated the file only has: | ||
313 | + "hash:" <filename> "-" <md5sum> | ||
314 | + ...inside it. | ||
315 | + | ||
316 | + Deflate is run before you add the big file to the index for committing. | ||
317 | + Deflate will replace the file contents with a hash, and copy the | ||
318 | + real content to your local blimpstore: | ||
319 | + | ||
320 | + "blimpstore"/<filename>-<md5sum> | ||
321 | + | ||
322 | + ...and if configured also upload it to "remote", using rsync. | ||
323 | + | ||
324 | + Configuration is in these locations in order: | ||
325 | + | ||
326 | + ./.blimp.conf | ||
327 | + "gitroot"/.blimp.conf | ||
328 | + ~/<blimpstore>/.blimp.conf | ||
329 | + ~/.blimp.conf | ||
330 | + | ||
331 | + This way you can have settings per directory, per git clone, | ||
332 | + per store and per user. A default blimpstore with a commented .blimp.conf | ||
333 | + is created in ~/blimpstore if you run blimp and no .blimp.conf is found. | ||
334 | + | ||
335 | + Edit ~/blimpstore/.blimp.conf (or in another location) and set a proper | ||
336 | + remote and the proper rsync password. This ensures that its also properly | ||
337 | + synced with a master rsync repository that is typically shared. | ||
338 | + | ||
339 | + Inflate will bring back the original content by copying from | ||
340 | + your local blimpstore, and if its not there, first downloading from the remote. | ||
341 | + Use this whenever you need to work/edit the big file - in order to get | ||
342 | + its real content. | ||
343 | + | ||
344 | + The filenames given are all processed. If -s is used content is processed via | ||
345 | + stdin/stdout and only one filename can be passed. This is used when running blimp | ||
346 | + via a git filter (smudge/clean). | ||
347 | + | ||
348 | + The remove command (no single character shortcut) will remove the file(s) content | ||
349 | + both from the local blimpstore and from the remote. This only removes | ||
350 | + the current content version, not older versions. The file itself is first | ||
351 | + inflated, if needed, and not deleted. This only "unblimps" the file. | ||
352 | +""" | ||
284 | 353 | ||
285 | ################################ main ##################################### | 354 | ################################ main ##################################### |
286 | # Set some dirs | 355 | # Set some dirs |
@@ -292,6 +361,18 @@ gitRootDir = gitRoot() | @@ -292,6 +361,18 @@ gitRootDir = gitRoot() | ||
292 | # Using lapp to get args, on parsing failure this will show usage automatically | 361 | # Using lapp to get args, on parsing failure this will show usage automatically |
293 | var args = parse(help) | 362 | var args = parse(help) |
294 | verbose = args["verbose"].asBool | 363 | verbose = args["verbose"].asBool |
364 | +stdio = args["stdio"].asBool | ||
365 | + | ||
366 | +# Can't do verbose with -s, that messes up stdout, | ||
367 | +# read in all of stdin once and for all | ||
368 | +if stdio: | ||
369 | + verbose = false | ||
370 | + try: | ||
371 | + stdinContent = readAll(stdin) | ||
372 | + close(stdin) | ||
373 | + except: | ||
374 | + quit("Failed reading stdin", 1) | ||
375 | + | ||
295 | 376 | ||
296 | # Parse configuration files, may shadow and override each other | 377 | # Parse configuration files, may shadow and override each other |
297 | parseConfFile(currentDir / ".blimp.conf") | 378 | parseConfFile(currentDir / ".blimp.conf") |
@@ -312,6 +393,10 @@ if verbose: dumpConfig() | @@ -312,6 +393,10 @@ if verbose: dumpConfig() | ||
312 | if args.showHelp: quit(help) | 393 | if args.showHelp: quit(help) |
313 | if args.showVersion: quit("blimp version: " & versionAsString) | 394 | if args.showVersion: quit("blimp version: " & versionAsString) |
314 | 395 | ||
396 | +# Check blimpVersion | ||
397 | +if not blimpVersion.isNil and blimpVersion != versionAsString: | ||
398 | + quit("Wrong version of blimp, configuration wants: " & blimpVersion) | ||
399 | + | ||
315 | let command = args["command"].asString | 400 | let command = args["command"].asString |
316 | let filenames = args["filenames"].asSeq | 401 | let filenames = args["filenames"].asSeq |
317 | 402 |
blimp.nimble
1 | [Package] | 1 | [Package] |
2 | name = "blimp" | 2 | name = "blimp" |
3 | -version = "0.2" | 3 | +version = "0.3" |
4 | author = "Göran Krampe" | 4 | author = "Göran Krampe" |
5 | description = "Utility that helps with big files in git, very similar to git-fat, s3annnex etc." | 5 | description = "Utility that helps with big files in git, very similar to git-fat, s3annnex etc." |
6 | license = "MIT" | 6 | license = "MIT" |