Commit ba6047f599fa385e2728f8ec1139418d4cfb98b1
1 parent
3e2f4c14
Added areas, moved to scp, added upload/download and various fixes.
Showing
2 changed files
with
150 additions
and
101 deletions
blimp.nim
... | ... | @@ -2,30 +2,49 @@ import md5, os, osproc, parseopt2, strutils, parsecfg, streams, lapp, subexes, t |
2 | 2 | |
3 | 3 | # blimp is a little utility program for handling large files |
4 | 4 | # in git repositories. Its inspired by git-fat and s3annex |
5 | -# but doesn't rely on S3 for storage - it uses rsync like git-fat. | |
5 | +# but doesn't rely on S3 for storage - it uses sftp but | |
6 | +# performs the file operations using externally configured commands. | |
6 | 7 | # It is a single binary without any dependencies. |
7 | 8 | # |
9 | +# It can also keep track of multiple remote areas and do up/downloads | |
10 | +# independent from git. This is useful in build scripts, up and | |
11 | +# downloading artifacts. | |
12 | +# | |
8 | 13 | # Just run blimp --help for detailed help. |
9 | 14 | |
10 | 15 | const |
11 | 16 | versionMajor* = 0 |
12 | - versionMinor* = 3 | |
17 | + versionMinor* = 4 | |
13 | 18 | versionPatch* = 0 |
14 | 19 | versionAsString* = $versionMajor & "." & $versionMinor & "." & $versionPatch |
15 | 20 | |
21 | +type | |
22 | + RemoteArea = ref object of RootObj | |
23 | + name*: string | |
24 | + url*: string | |
25 | + upload*: string | |
26 | + download*: string | |
27 | + delete*: string | |
28 | + | |
16 | 29 | var |
17 | - blimpStore, remoteBlimpStore, uploadCommandFormat, downloadCommandFormat, deleteCommandFormat, rsyncPassword, blimpVersion: string = nil | |
30 | + blimpStore, blimpVersion: string = nil | |
18 | 31 | homeDir, currentDir, gitRootDir: string |
19 | 32 | verbose, stdio, onAllDeflated, onAllFiltered: bool |
20 | - stdinContent: string = nil | |
33 | + stdinContent, area: string = nil | |
34 | + areas = newTable[string, RemoteArea]() | |
35 | + remoteArea: RemoteArea | |
21 | 36 | |
22 | 37 | let |
23 | 38 | defaultConfig = """ |
24 | -[rsync] | |
39 | +[blimp] | |
40 | +# Minimal version, otherwise stop | |
41 | +# version = """ & versionAsString & """ | |
42 | + | |
25 | 43 | # Set your local blimpstore directory. You can use %home%, %cwd% and %gitroot% in paths, works cross platform. |
26 | 44 | # Example: |
27 | 45 | # # Place it inside the git clone |
28 | 46 | # blimpstore = "%gitroot%/.git/blimpstore" |
47 | +# | |
29 | 48 | # # Place it in current working directory (not very useful) |
30 | 49 | # blimpstore = "%cwd%/blimpstore" |
31 | 50 | # |
... | ... | @@ -33,27 +52,28 @@ let |
33 | 52 | # # Place it in the users home directory |
34 | 53 | # blimpstore = "%home%/blimpstore" |
35 | 54 | |
36 | -# Set this to your remote rsync location | |
37 | -remote = "blimpuser@some-rsync-server.com::blimpstore" | |
38 | -# Set this to your rsync password, it will be written out as a password-file called .blimp.pass on every rsync. | |
39 | -password = "some-good-rsync-password-for-blimpuser" | |
40 | - | |
41 | -# The following three formats should not need editing. | |
42 | -# $1 is the blimp filename, $2 is remote location and $3 is the local blimpstore directory set above. | |
43 | -# NOTE: The password-file .blimp.pass will be created by blimp on every command, do not remove that option. | |
44 | -upload = "rsync --password-file $3/.blimp.pass -avzP $3/$1 $2/" | |
45 | -download = "rsync --password-file $3/.blimp.pass -avzP $2/$1 $3/" | |
46 | -# This deletes a single file from destination, that is already deleted in source. Yeah... insane! But it works. | |
47 | -delete = "rsync --password-file $3/.blimp.pass -dv --delete --existing --ignore-existing --include '$1' --exclude '*' $3/ $2" | |
48 | - | |
49 | -[blimp] | |
50 | -# Minimal version, otherwise stop | |
51 | -# version = """ & versionAsString | |
55 | +[areas] | |
56 | +# This is where ypu define up/download areas. The area called "remote" is the | |
57 | +# default area used unless --area is used. | |
58 | +# $1 is the blimp filename, $2 is the remote url and $3 is the local blimpstore directory set above. | |
59 | +remote-url = "blimpuser@some-server.com:/var/opt/blimpstore" | |
60 | +remote-upload = "scp -pq $3/$1 $2/" | |
61 | +remote-download = "scp -pq $2/$1 $3/" | |
52 | 62 | |
63 | +# Example area that can be used with upload/download commands and --area option. | |
64 | +release-url = "blimpuser@some-server.com:/var/opt/release" | |
65 | +release-upload = "scp -r $3/$1 $2/" | |
66 | +release-download = "scp -r $2/$1 $3/" | |
67 | +""" | |
53 | 68 | |
54 | 69 | proc cmd(cmd: string): string = |
55 | 70 | try: |
56 | - let tup = execCmdEx(cmd) | |
71 | + # Otherwise pipes will not work for git commands etc | |
72 | + when defined(windows): | |
73 | + let tup = execCmdEx("cmd /c \"" & cmd & "\"") | |
74 | + else: | |
75 | + let tup = execCmdEx(cmd) | |
76 | + #echo "cmd: " & $cmd & "err:" & $tup[1] | |
57 | 77 | if tup[1] == 0: |
58 | 78 | result = strip(tup[0]) |
59 | 79 | else: |
... | ... | @@ -99,31 +119,41 @@ proc parseConfFile(filename: string) = |
99 | 119 | if verbose: echo "Reading config: " & filename |
100 | 120 | var p: CfgParser |
101 | 121 | open(p, f, filename) |
122 | + var section: string | |
123 | + var area: RemoteArea | |
102 | 124 | while true: |
103 | 125 | var e = next(p) |
104 | 126 | case e.kind |
105 | 127 | of cfgEof: |
106 | 128 | break |
107 | 129 | of cfgSectionStart: |
108 | - continue # Ignore | |
130 | + section = e.section | |
109 | 131 | of cfgKeyValuePair: |
110 | - case e.key | |
111 | - of "blimpstore": | |
112 | - if blimpStore.isNil: blimpStore = expandDirs(e.value) | |
113 | - of "remote": | |
114 | - if remoteBlimpStore.isNil: remoteBlimpStore = expandDirs(e.value) | |
115 | - of "password": | |
116 | - if rsyncPassword.isNil: rsyncPassword = e.value | |
117 | - of "upload": | |
118 | - if uploadCommandFormat.isNil: uploadCommandFormat = expandDirs(e.value) | |
119 | - of "download": | |
120 | - if downloadCommandFormat.isNil: downloadCommandFormat = expandDirs(e.value) | |
121 | - of "delete": | |
122 | - if deleteCommandFormat.isNil: deleteCommandFormat = expandDirs(e.value) | |
123 | - of "version": | |
124 | - if blimpVersion.isNil: blimpVersion = e.value | |
132 | + case section | |
133 | + of "blimp": | |
134 | + case e.key | |
135 | + of "blimpstore": | |
136 | + if blimpStore.isNil: blimpStore = e.value | |
137 | + of "version": | |
138 | + if blimpVersion.isNil: blimpVersion = e.value | |
139 | + else: | |
140 | + quit("Unknown blimp configuration: " & e.key) | |
125 | 141 | else: |
126 | - quit("Unknown configuration: " & e.key) | |
142 | + # Then we presume its an area | |
143 | + if area.isNil or area.name != section: | |
144 | + area = RemoteArea(name: section) | |
145 | + areas[area.name] = area | |
146 | + case e.key | |
147 | + of "url": | |
148 | + if area.url.isNil: area.url = expandDirs(e.value) | |
149 | + of "upload": | |
150 | + if area.upload.isNil: area.upload = expandDirs(e.value) | |
151 | + of "download": | |
152 | + if area.download.isNil: area.download = expandDirs(e.value) | |
153 | + of "delete": | |
154 | + if area.delete.isNil: area.delete = expandDirs(e.value) | |
155 | + else: | |
156 | + quit("Unknown area configuration: " & e.key) | |
127 | 157 | of cfgOption: |
128 | 158 | quit("Unknown configuration: " & e.key) |
129 | 159 | of cfgError: |
... | ... | @@ -131,42 +161,35 @@ proc parseConfFile(filename: string) = |
131 | 161 | close(p) |
132 | 162 | |
133 | 163 | # Trivial helper to enable verbose |
134 | -proc run(cmd: string): auto = | |
164 | +proc run(cmd: string): int = | |
135 | 165 | if verbose: echo(cmd) |
136 | 166 | execCmd(cmd) |
137 | 167 | |
138 | -# Every rsync command, make sure we have a password file | |
139 | -proc rsyncRun(cmd: string): auto = | |
140 | - if not rsyncPassword.isNil: | |
141 | - writeFile(blimpStore / ".blimp.pass", rsyncPassword) | |
142 | - if execCmd("chmod 600 " & blimpStore / ".blimp.pass") != 0: | |
143 | - quit("Failed to chmod 600 " & blimpStore / ".blimp.pass") | |
144 | - run(cmd) | |
145 | - | |
146 | -# Upload a file to the remote master blimpStore | |
147 | -proc uploadFile(blimpFilename: string) = | |
148 | - if remoteBlimpStore.isNil: | |
149 | - echo("Remote blimpstore not set in configuration file, skipping uploading content:\n\t" & blimpFilename) | |
168 | + | |
169 | +# Upload a file to the remote area | |
170 | +proc uploadFile(filename, fromDir: string) = | |
171 | + if remoteArea.isNil: | |
172 | + echo("Remote area not set in configuration file, skipping uploading content:\n\t" & filename) | |
150 | 173 | return |
151 | - let errorCode = rsyncRun(format(uploadCommandFormat, blimpFilename, remoteBlimpStore, blimpStore)) | |
174 | + let errorCode = run(format(remoteArea.upload, filename, remoteArea.url, fromDir)) | |
152 | 175 | if errorCode != 0: |
153 | - quit("Something went wrong uploading " & blimpFilename & " to " & remoteBlimpStore, 2) | |
176 | + quit("Something went wrong uploading " & filename & " to " & remoteArea.url, 2) | |
154 | 177 | |
155 | -# Download a file to the remote master blimpStore | |
156 | -proc downloadFile(blimpFilename: string) = | |
157 | - if remoteBlimpStore.isNil: | |
158 | - quit("Remote blimpstore not set in configuration file, can not download content:\n\t" & blimpFilename) | |
159 | - let errorCode = rsyncRun(format(downloadCommandFormat, blimpFilename, remoteBlimpStore, blimpStore)) | |
178 | +# Download a file to the remote area | |
179 | +proc downloadFile(filename, toDir: string) = | |
180 | + if remoteArea.isNil: | |
181 | + quit("Remote area not set in configuration file, can not download content:\n\t" & filename) | |
182 | + let errorCode = run(format(remoteArea.download, filename, remoteArea.url, toDir)) | |
160 | 183 | if errorCode != 0: |
161 | - quit("Something went wrong downloading " & blimpFilename & " from " & remoteBlimpStore, 3) | |
184 | + quit("Something went wrong downloading " & filename & " from " & remoteArea.url, 3) | |
162 | 185 | |
163 | 186 | # Delete a file from the remote master blimpStore |
164 | -proc remoteDeleteFile(blimpFilename: string) = | |
165 | - if remoteBlimpStore.isNil: | |
187 | +proc remoteDeleteFile(filename: string) = | |
188 | + if remoteArea.isNil: | |
166 | 189 | return |
167 | - let errorCode = rsyncRun(format(deleteCommandFormat, blimpFilename, remoteBlimpStore, blimpStore)) | |
190 | + let errorCode = run(format(remoteArea.delete, filename, remoteArea.url, blimpStore)) | |
168 | 191 | if errorCode != 0: |
169 | - quit("Something went wrong deleting " & blimpFilename & " from " & remoteBlimpStore, 3) | |
192 | + quit("Something went wrong deleting " & filename & " from " & remoteArea.url, 3) | |
170 | 193 | |
171 | 194 | # Copy content to blimpStore and upload if it was a new file or upload == true. |
172 | 195 | proc copyToBlimpStore(filename, blimpFilename: string) = |
... | ... | @@ -178,13 +201,13 @@ proc copyToBlimpStore(filename, blimpFilename: string) = |
178 | 201 | quit("Failed writing file: " & blimpStore / blimpFilename & " from stdin", 1) |
179 | 202 | else: |
180 | 203 | copyFile(filename, blimpStore / blimpFilename) |
181 | - uploadFile(blimpFilename) | |
204 | + uploadFile(blimpFilename, blimpStore) | |
182 | 205 | |
183 | 206 | |
184 | 207 | # Copy content from blimpStore, and downloading first if needed |
185 | 208 | proc copyFromBlimpStore(blimpFilename, filename: string) = |
186 | 209 | if not existsFile(blimpStore / blimpFilename): |
187 | - downloadFile(blimpFilename) | |
210 | + downloadFile(blimpFilename, blimpStore) | |
188 | 211 | if stdio: |
189 | 212 | try: |
190 | 213 | var content = readFile(blimpStore / blimpFilename) |
... | ... | @@ -279,18 +302,29 @@ proc remove(filename: string) = |
279 | 302 | echo("\t" & filename & " content removed from blimpstore locally and remotely.") |
280 | 303 | |
281 | 304 | |
282 | -# Copy original file to blimpStore and replace with hash stub in git. | |
283 | -proc upload(filename: string) = | |
305 | +# Make sure a file already in blimpstore is uploaded to remote area. | |
306 | +proc push(filename: string) = | |
284 | 307 | if verbose: echo "Uploading " & filename |
285 | 308 | var blimpFilename = blimpFilename(filename) |
286 | 309 | if blimpFilename.isNil: |
287 | 310 | blimpFilename = computeBlimpFilename(filename) |
288 | 311 | if existsFile(blimpStore / blimpFilename): |
289 | - uploadFile(blimpFilename) | |
312 | + uploadFile(blimpFilename, blimpStore) | |
290 | 313 | if verbose: echo("\t" & filename & " uploaded.") |
291 | 314 | else: |
292 | 315 | if verbose: echo("\t" & filename & " is not in blimpstore, skipping.") |
293 | 316 | |
317 | +# Upload a file to a remote area | |
318 | +proc upload(filename: string) = | |
319 | + if verbose: echo "Uploading " & filename | |
320 | + uploadFile(filename, currentDir) | |
321 | + if verbose: echo("\t" & filename & " uploaded.") | |
322 | + | |
323 | +# Download a file from a remote area. | |
324 | +proc download(filename: string) = | |
325 | + if verbose: echo "Downloading " & filename | |
326 | + downloadFile(filename, currentDir) | |
327 | + if verbose: echo("\t" & filename & " downloaded.") | |
294 | 328 | |
295 | 329 | proc setupBlimpStore() = |
296 | 330 | try: |
... | ... | @@ -311,27 +345,25 @@ proc `$`(x: string): string = |
311 | 345 | proc dumpConfig() = |
312 | 346 | echo "\nDump of configuration:" |
313 | 347 | echo "\tblimpStore: " & blimpStore |
314 | - echo "\tremoteBlimpStore: " & remoteBlimpStore | |
315 | - echo "\tuploadCommandFormat: " & uploadCommandFormat | |
316 | - echo "\tdownloadCommandFormat: " & downloadCommandFormat | |
317 | - echo "\tdeleteCommandFormat: " & deleteCommandFormat | |
318 | - echo "\trsyncPassword: " & $rsyncPassword | |
348 | + echo "\tremote-url: " & remoteArea.url | |
349 | + echo "\tremote-upload: " & remoteArea.upload | |
350 | + echo "\tremote-download: " & remoteArea.download | |
319 | 351 | echo "\tblimpVersion: " & $blimpVersion |
320 | 352 | echo "\n" |
321 | 353 | |
322 | 354 | let synopsis = """ |
323 | 355 | blimp [options] <command> <filenames...> |
324 | - -h,--help Show this | |
325 | - --version Show version of blimp | |
326 | - -v,--verbose Verbosity, only works without -s | |
327 | - -i,--init Set blimp filter in git config | |
356 | + -h,--help Show this | |
357 | + --version Show version of blimp | |
358 | + -v,--verbose Verbosity, only works without -s | |
328 | 359 | ---------- |
329 | - <command> (string) (d)eflate, (i)nflate, remove, upload | |
330 | - -a,--all Operate on all deflated files in clone | |
331 | - -f,--filter Operate on all files matching blimp filter | |
360 | + <command> (string) (d)eflate, (i)nflate, init, remove, push, upload, download | |
361 | + --all Operate on all deflated files in clone | |
362 | + -f,--filter Operate on all files matching blimp filter | |
363 | + -a,--area (default remote) The area to use for remote up/downloads | |
332 | 364 | ---------- |
333 | - -s,--stdio If given, use stdin/stdout for content. | |
334 | - <filenames> (string...) One or more filepaths to inflate/deflate | |
365 | + -s,--stdio If given, use stdin/stdout for content. | |
366 | + <filenames> (string...) One or more filepaths to inflate/deflate | |
335 | 367 | """ |
336 | 368 | let help = """ |
337 | 369 | |
... | ... | @@ -385,9 +417,12 @@ let help = """ |
385 | 417 | the current content version, not older versions. The file itself is first |
386 | 418 | inflated, if needed, and not deleted. This only "unblimps" the file. |
387 | 419 | |
388 | - The upload command (no single character shortcut) will upload the given file | |
420 | + The push command (no single character shortcut) will force upload the given file | |
389 | 421 | from the local blimpstore to the remote. This is normally done automatically, |
390 | - but this way you can make sure they are up on the remote. | |
422 | + but this way you can make sure they are synced onto the remote. | |
423 | + | |
424 | + The upload and download commands are used to distribute artifacts typically in a | |
425 | + build script. If no --area is given, they use the standard "remote" area. | |
391 | 426 | |
392 | 427 | In order to have blimp work automatically you can: |
393 | 428 | |
... | ... | @@ -398,7 +433,7 @@ let help = """ |
398 | 433 | git config filter.blimp.clean "blimp -s d %f" |
399 | 434 | git config filter.blimp.smudge "blimp -s i %f" |
400 | 435 | |
401 | - The above git config can be done by running "blimp --init". | |
436 | + The above git config can be done by running "blimp init" just like git. | |
402 | 437 | |
403 | 438 | When the above is done (per clone) git will automatically run blimp deflate |
404 | 439 | just before committing and blimp inflate when operations are done. |
... | ... | @@ -406,9 +441,10 @@ let help = """ |
406 | 441 | This means that if you clone a git repository that already has a .gitattributes |
407 | 442 | file in it that uses the blimp filter, then you should do: |
408 | 443 | |
409 | - blimp --init inflate --filter | |
444 | + blimp init | |
445 | + blimp inflate --filter | |
410 | 446 | |
411 | - This will configure the blimp filter and also find and inflate all deflated | |
447 | + This will configure the blimp filter and then find and inflate all deflated | |
412 | 448 | files throughout the clone. |
413 | 449 | """ |
414 | 450 | |
... | ... | @@ -425,6 +461,7 @@ verbose = args["verbose"].asBool |
425 | 461 | stdio = args["stdio"].asBool |
426 | 462 | onAllDeflated = args["all"].asBool |
427 | 463 | onAllFiltered = args["filter"].asBool |
464 | +area = args["area"].asString | |
428 | 465 | |
429 | 466 | # Can't do verbose with -s, that messes up stdout, |
430 | 467 | # read in all of stdin once and for all |
... | ... | @@ -436,19 +473,23 @@ if stdio: |
436 | 473 | except: |
437 | 474 | quit("Failed reading stdin", 1) |
438 | 475 | |
439 | - | |
440 | 476 | # Parse configuration files, may shadow and override each other |
441 | 477 | parseConfFile(currentDir / ".blimp.conf") |
442 | 478 | if not gitRootDir.isNil and gitRootDir != currentDir: |
443 | 479 | parseConfFile(gitRootDir / ".blimp.conf") |
444 | 480 | |
481 | +if existsDir(homeDir / "blimpstore"): | |
482 | + parseConfFile(homeDir / "blimpstore" / ".blimp.conf") | |
483 | + | |
484 | +parseConfFile(homeDir / ".blimp.conf") | |
485 | + | |
445 | 486 | # If we haven't gotten a blimpstore yet, we set a default one |
446 | 487 | if blimpStore.isNil: |
447 | 488 | blimpStore = homeDir / "blimpstore" |
448 | 489 | |
449 | -if existsDir(blimpStore): | |
450 | - parseConfFile(blimpStore / ".blimp.conf") | |
451 | -parseConfFile(homeDir / ".blimp.conf") | |
490 | +# Check if we have a configured remoteArea | |
491 | +if areas.hasKey(area): | |
492 | + remoteArea = areas[area] | |
452 | 493 | |
453 | 494 | # Let's just show what we have :) |
454 | 495 | if verbose: dumpConfig() |
... | ... | @@ -461,12 +502,8 @@ if args.showVersion: quit("blimp version: " & versionAsString) |
461 | 502 | if not blimpVersion.isNil and blimpVersion != versionAsString: |
462 | 503 | quit("Wrong version of blimp, configuration wants: " & blimpVersion) |
463 | 504 | |
464 | -# Should we install the filter? | |
465 | -if args["init"].asBool: | |
466 | - ensureBlimpFilter() | |
467 | - if verbose: echo("Installed blimp filter") | |
468 | 505 | |
469 | -let command = args["command"].asString | |
506 | +# Ok, let's see | |
470 | 507 | var filenames = initSet[string]() |
471 | 508 | |
472 | 509 | # Add upp all files to operate on in a Set |
... | ... | @@ -484,9 +521,16 @@ if onAllFiltered: |
484 | 521 | # Make sure the local blimpstore is setup. |
485 | 522 | setupBlimpStore() |
486 | 523 | |
524 | + | |
487 | 525 | # Do the deed |
526 | +let command = args["command"].asString | |
488 | 527 | if command != "": |
489 | - if command == "d" or command == "deflate": | |
528 | + # Should we install the filter? | |
529 | + if command == "init": | |
530 | + ensureBlimpFilter() | |
531 | + echo("Installed blimp filter") | |
532 | + quit(0) | |
533 | + elif command == "d" or command == "deflate": | |
490 | 534 | for fn in filenames: |
491 | 535 | deflate(fn) |
492 | 536 | elif command == "i" or command == "inflate": |
... | ... | @@ -495,12 +539,17 @@ if command != "": |
495 | 539 | elif command == "remove": |
496 | 540 | for fn in filenames: |
497 | 541 | remove(fn) |
542 | + elif command == "push": | |
543 | + for fn in filenames.items: | |
544 | + push(fn) | |
498 | 545 | elif command == "upload": |
499 | - echo repr(filenames) | |
500 | 546 | for fn in filenames.items: |
501 | 547 | upload(fn) |
548 | + elif command == "download": | |
549 | + for fn in filenames.items: | |
550 | + download(fn) | |
502 | 551 | else: |
503 | - quit("Unknown command: \"" & command & "\", only (d)eflate, (i)inflate, remove or upload are valid.", 6) | |
552 | + quit("Unknown command: \"" & command & "\", use --help for valid commands.", 6) | |
504 | 553 | |
505 | 554 | # All good |
506 | 555 | quit(0) | ... | ... |