Commit ba6047f599fa385e2728f8ec1139418d4cfb98b1

Authored by Göran Krampe
1 parent 3e2f4c14

Added areas, moved to scp, added upload/download and various fixes.

Showing 2 changed files with 150 additions and 101 deletions
blimp.nim
... ... @@ -2,30 +2,49 @@ import md5, os, osproc, parseopt2, strutils, parsecfg, streams, lapp, subexes, t
2 2  
3 3 # blimp is a little utility program for handling large files
4 4 # in git repositories. Its inspired by git-fat and s3annex
5   -# but doesn't rely on S3 for storage - it uses rsync like git-fat.
  5 +# but doesn't rely on S3 for storage - it uses sftp but
  6 +# performs the file operations using externally configured commands.
6 7 # It is a single binary without any dependencies.
7 8 #
  9 +# It can also keep track of multiple remote areas and do up/downloads
  10 +# independent from git. This is useful in build scripts, up and
  11 +# downloading artifacts.
  12 +#
8 13 # Just run blimp --help for detailed help.
9 14  
10 15 const
11 16 versionMajor* = 0
12   - versionMinor* = 3
  17 + versionMinor* = 4
13 18 versionPatch* = 0
14 19 versionAsString* = $versionMajor & "." & $versionMinor & "." & $versionPatch
15 20  
  21 +type
  22 + RemoteArea = ref object of RootObj
  23 + name*: string
  24 + url*: string
  25 + upload*: string
  26 + download*: string
  27 + delete*: string
  28 +
16 29 var
17   - blimpStore, remoteBlimpStore, uploadCommandFormat, downloadCommandFormat, deleteCommandFormat, rsyncPassword, blimpVersion: string = nil
  30 + blimpStore, blimpVersion: string = nil
18 31 homeDir, currentDir, gitRootDir: string
19 32 verbose, stdio, onAllDeflated, onAllFiltered: bool
20   - stdinContent: string = nil
  33 + stdinContent, area: string = nil
  34 + areas = newTable[string, RemoteArea]()
  35 + remoteArea: RemoteArea
21 36  
22 37 let
23 38 defaultConfig = """
24   -[rsync]
  39 +[blimp]
  40 +# Minimal version, otherwise stop
  41 +# version = """ & versionAsString & """
  42 +
25 43 # Set your local blimpstore directory. You can use %home%, %cwd% and %gitroot% in paths, works cross platform.
26 44 # Example:
27 45 # # Place it inside the git clone
28 46 # blimpstore = "%gitroot%/.git/blimpstore"
  47 +#
29 48 # # Place it in current working directory (not very useful)
30 49 # blimpstore = "%cwd%/blimpstore"
31 50 #
... ... @@ -33,27 +52,28 @@ let
33 52 # # Place it in the users home directory
34 53 # blimpstore = "%home%/blimpstore"
35 54  
36   -# Set this to your remote rsync location
37   -remote = "blimpuser@some-rsync-server.com::blimpstore"
38   -# Set this to your rsync password, it will be written out as a password-file called .blimp.pass on every rsync.
39   -password = "some-good-rsync-password-for-blimpuser"
40   -
41   -# The following three formats should not need editing.
42   -# $1 is the blimp filename, $2 is remote location and $3 is the local blimpstore directory set above.
43   -# NOTE: The password-file .blimp.pass will be created by blimp on every command, do not remove that option.
44   -upload = "rsync --password-file $3/.blimp.pass -avzP $3/$1 $2/"
45   -download = "rsync --password-file $3/.blimp.pass -avzP $2/$1 $3/"
46   -# This deletes a single file from destination, that is already deleted in source. Yeah... insane! But it works.
47   -delete = "rsync --password-file $3/.blimp.pass -dv --delete --existing --ignore-existing --include '$1' --exclude '*' $3/ $2"
48   -
49   -[blimp]
50   -# Minimal version, otherwise stop
51   -# version = """ & versionAsString
  55 +[areas]
  56 +# This is where ypu define up/download areas. The area called "remote" is the
  57 +# default area used unless --area is used.
  58 +# $1 is the blimp filename, $2 is the remote url and $3 is the local blimpstore directory set above.
  59 +remote-url = "blimpuser@some-server.com:/var/opt/blimpstore"
  60 +remote-upload = "scp -pq $3/$1 $2/"
  61 +remote-download = "scp -pq $2/$1 $3/"
52 62  
  63 +# Example area that can be used with upload/download commands and --area option.
  64 +release-url = "blimpuser@some-server.com:/var/opt/release"
  65 +release-upload = "scp -r $3/$1 $2/"
  66 +release-download = "scp -r $2/$1 $3/"
  67 +"""
53 68  
54 69 proc cmd(cmd: string): string =
55 70 try:
56   - let tup = execCmdEx(cmd)
  71 + # Otherwise pipes will not work for git commands etc
  72 + when defined(windows):
  73 + let tup = execCmdEx("cmd /c \"" & cmd & "\"")
  74 + else:
  75 + let tup = execCmdEx(cmd)
  76 + #echo "cmd: " & $cmd & "err:" & $tup[1]
57 77 if tup[1] == 0:
58 78 result = strip(tup[0])
59 79 else:
... ... @@ -99,31 +119,41 @@ proc parseConfFile(filename: string) =
99 119 if verbose: echo "Reading config: " & filename
100 120 var p: CfgParser
101 121 open(p, f, filename)
  122 + var section: string
  123 + var area: RemoteArea
102 124 while true:
103 125 var e = next(p)
104 126 case e.kind
105 127 of cfgEof:
106 128 break
107 129 of cfgSectionStart:
108   - continue # Ignore
  130 + section = e.section
109 131 of cfgKeyValuePair:
110   - case e.key
111   - of "blimpstore":
112   - if blimpStore.isNil: blimpStore = expandDirs(e.value)
113   - of "remote":
114   - if remoteBlimpStore.isNil: remoteBlimpStore = expandDirs(e.value)
115   - of "password":
116   - if rsyncPassword.isNil: rsyncPassword = e.value
117   - of "upload":
118   - if uploadCommandFormat.isNil: uploadCommandFormat = expandDirs(e.value)
119   - of "download":
120   - if downloadCommandFormat.isNil: downloadCommandFormat = expandDirs(e.value)
121   - of "delete":
122   - if deleteCommandFormat.isNil: deleteCommandFormat = expandDirs(e.value)
123   - of "version":
124   - if blimpVersion.isNil: blimpVersion = e.value
  132 + case section
  133 + of "blimp":
  134 + case e.key
  135 + of "blimpstore":
  136 + if blimpStore.isNil: blimpStore = e.value
  137 + of "version":
  138 + if blimpVersion.isNil: blimpVersion = e.value
  139 + else:
  140 + quit("Unknown blimp configuration: " & e.key)
125 141 else:
126   - quit("Unknown configuration: " & e.key)
  142 + # Then we presume its an area
  143 + if area.isNil or area.name != section:
  144 + area = RemoteArea(name: section)
  145 + areas[area.name] = area
  146 + case e.key
  147 + of "url":
  148 + if area.url.isNil: area.url = expandDirs(e.value)
  149 + of "upload":
  150 + if area.upload.isNil: area.upload = expandDirs(e.value)
  151 + of "download":
  152 + if area.download.isNil: area.download = expandDirs(e.value)
  153 + of "delete":
  154 + if area.delete.isNil: area.delete = expandDirs(e.value)
  155 + else:
  156 + quit("Unknown area configuration: " & e.key)
127 157 of cfgOption:
128 158 quit("Unknown configuration: " & e.key)
129 159 of cfgError:
... ... @@ -131,42 +161,35 @@ proc parseConfFile(filename: string) =
131 161 close(p)
132 162  
133 163 # Trivial helper to enable verbose
134   -proc run(cmd: string): auto =
  164 +proc run(cmd: string): int =
135 165 if verbose: echo(cmd)
136 166 execCmd(cmd)
137 167  
138   -# Every rsync command, make sure we have a password file
139   -proc rsyncRun(cmd: string): auto =
140   - if not rsyncPassword.isNil:
141   - writeFile(blimpStore / ".blimp.pass", rsyncPassword)
142   - if execCmd("chmod 600 " & blimpStore / ".blimp.pass") != 0:
143   - quit("Failed to chmod 600 " & blimpStore / ".blimp.pass")
144   - run(cmd)
145   -
146   -# Upload a file to the remote master blimpStore
147   -proc uploadFile(blimpFilename: string) =
148   - if remoteBlimpStore.isNil:
149   - echo("Remote blimpstore not set in configuration file, skipping uploading content:\n\t" & blimpFilename)
  168 +
  169 +# Upload a file to the remote area
  170 +proc uploadFile(filename, fromDir: string) =
  171 + if remoteArea.isNil:
  172 + echo("Remote area not set in configuration file, skipping uploading content:\n\t" & filename)
150 173 return
151   - let errorCode = rsyncRun(format(uploadCommandFormat, blimpFilename, remoteBlimpStore, blimpStore))
  174 + let errorCode = run(format(remoteArea.upload, filename, remoteArea.url, fromDir))
152 175 if errorCode != 0:
153   - quit("Something went wrong uploading " & blimpFilename & " to " & remoteBlimpStore, 2)
  176 + quit("Something went wrong uploading " & filename & " to " & remoteArea.url, 2)
154 177  
155   -# Download a file to the remote master blimpStore
156   -proc downloadFile(blimpFilename: string) =
157   - if remoteBlimpStore.isNil:
158   - quit("Remote blimpstore not set in configuration file, can not download content:\n\t" & blimpFilename)
159   - let errorCode = rsyncRun(format(downloadCommandFormat, blimpFilename, remoteBlimpStore, blimpStore))
  178 +# Download a file to the remote area
  179 +proc downloadFile(filename, toDir: string) =
  180 + if remoteArea.isNil:
  181 + quit("Remote area not set in configuration file, can not download content:\n\t" & filename)
  182 + let errorCode = run(format(remoteArea.download, filename, remoteArea.url, toDir))
160 183 if errorCode != 0:
161   - quit("Something went wrong downloading " & blimpFilename & " from " & remoteBlimpStore, 3)
  184 + quit("Something went wrong downloading " & filename & " from " & remoteArea.url, 3)
162 185  
163 186 # Delete a file from the remote master blimpStore
164   -proc remoteDeleteFile(blimpFilename: string) =
165   - if remoteBlimpStore.isNil:
  187 +proc remoteDeleteFile(filename: string) =
  188 + if remoteArea.isNil:
166 189 return
167   - let errorCode = rsyncRun(format(deleteCommandFormat, blimpFilename, remoteBlimpStore, blimpStore))
  190 + let errorCode = run(format(remoteArea.delete, filename, remoteArea.url, blimpStore))
168 191 if errorCode != 0:
169   - quit("Something went wrong deleting " & blimpFilename & " from " & remoteBlimpStore, 3)
  192 + quit("Something went wrong deleting " & filename & " from " & remoteArea.url, 3)
170 193  
171 194 # Copy content to blimpStore and upload if it was a new file or upload == true.
172 195 proc copyToBlimpStore(filename, blimpFilename: string) =
... ... @@ -178,13 +201,13 @@ proc copyToBlimpStore(filename, blimpFilename: string) =
178 201 quit("Failed writing file: " & blimpStore / blimpFilename & " from stdin", 1)
179 202 else:
180 203 copyFile(filename, blimpStore / blimpFilename)
181   - uploadFile(blimpFilename)
  204 + uploadFile(blimpFilename, blimpStore)
182 205  
183 206  
184 207 # Copy content from blimpStore, and downloading first if needed
185 208 proc copyFromBlimpStore(blimpFilename, filename: string) =
186 209 if not existsFile(blimpStore / blimpFilename):
187   - downloadFile(blimpFilename)
  210 + downloadFile(blimpFilename, blimpStore)
188 211 if stdio:
189 212 try:
190 213 var content = readFile(blimpStore / blimpFilename)
... ... @@ -279,18 +302,29 @@ proc remove(filename: string) =
279 302 echo("\t" & filename & " content removed from blimpstore locally and remotely.")
280 303  
281 304  
282   -# Copy original file to blimpStore and replace with hash stub in git.
283   -proc upload(filename: string) =
  305 +# Make sure a file already in blimpstore is uploaded to remote area.
  306 +proc push(filename: string) =
284 307 if verbose: echo "Uploading " & filename
285 308 var blimpFilename = blimpFilename(filename)
286 309 if blimpFilename.isNil:
287 310 blimpFilename = computeBlimpFilename(filename)
288 311 if existsFile(blimpStore / blimpFilename):
289   - uploadFile(blimpFilename)
  312 + uploadFile(blimpFilename, blimpStore)
290 313 if verbose: echo("\t" & filename & " uploaded.")
291 314 else:
292 315 if verbose: echo("\t" & filename & " is not in blimpstore, skipping.")
293 316  
  317 +# Upload a file to a remote area
  318 +proc upload(filename: string) =
  319 + if verbose: echo "Uploading " & filename
  320 + uploadFile(filename, currentDir)
  321 + if verbose: echo("\t" & filename & " uploaded.")
  322 +
  323 +# Download a file from a remote area.
  324 +proc download(filename: string) =
  325 + if verbose: echo "Downloading " & filename
  326 + downloadFile(filename, currentDir)
  327 + if verbose: echo("\t" & filename & " downloaded.")
294 328  
295 329 proc setupBlimpStore() =
296 330 try:
... ... @@ -311,27 +345,25 @@ proc `$`(x: string): string =
311 345 proc dumpConfig() =
312 346 echo "\nDump of configuration:"
313 347 echo "\tblimpStore: " & blimpStore
314   - echo "\tremoteBlimpStore: " & remoteBlimpStore
315   - echo "\tuploadCommandFormat: " & uploadCommandFormat
316   - echo "\tdownloadCommandFormat: " & downloadCommandFormat
317   - echo "\tdeleteCommandFormat: " & deleteCommandFormat
318   - echo "\trsyncPassword: " & $rsyncPassword
  348 + echo "\tremote-url: " & remoteArea.url
  349 + echo "\tremote-upload: " & remoteArea.upload
  350 + echo "\tremote-download: " & remoteArea.download
319 351 echo "\tblimpVersion: " & $blimpVersion
320 352 echo "\n"
321 353  
322 354 let synopsis = """
323 355 blimp [options] <command> <filenames...>
324   - -h,--help Show this
325   - --version Show version of blimp
326   - -v,--verbose Verbosity, only works without -s
327   - -i,--init Set blimp filter in git config
  356 + -h,--help Show this
  357 + --version Show version of blimp
  358 + -v,--verbose Verbosity, only works without -s
328 359 ----------
329   - <command> (string) (d)eflate, (i)nflate, remove, upload
330   - -a,--all Operate on all deflated files in clone
331   - -f,--filter Operate on all files matching blimp filter
  360 + <command> (string) (d)eflate, (i)nflate, init, remove, push, upload, download
  361 + --all Operate on all deflated files in clone
  362 + -f,--filter Operate on all files matching blimp filter
  363 + -a,--area (default remote) The area to use for remote up/downloads
332 364 ----------
333   - -s,--stdio If given, use stdin/stdout for content.
334   - <filenames> (string...) One or more filepaths to inflate/deflate
  365 + -s,--stdio If given, use stdin/stdout for content.
  366 + <filenames> (string...) One or more filepaths to inflate/deflate
335 367 """
336 368 let help = """
337 369  
... ... @@ -385,9 +417,12 @@ let help = &quot;&quot;&quot;
385 417 the current content version, not older versions. The file itself is first
386 418 inflated, if needed, and not deleted. This only "unblimps" the file.
387 419  
388   - The upload command (no single character shortcut) will upload the given file
  420 + The push command (no single character shortcut) will force upload the given file
389 421 from the local blimpstore to the remote. This is normally done automatically,
390   - but this way you can make sure they are up on the remote.
  422 + but this way you can make sure they are synced onto the remote.
  423 +
  424 + The upload and download commands are used to distribute artifacts typically in a
  425 + build script. If no --area is given, they use the standard "remote" area.
391 426  
392 427 In order to have blimp work automatically you can:
393 428  
... ... @@ -398,7 +433,7 @@ let help = &quot;&quot;&quot;
398 433 git config filter.blimp.clean "blimp -s d %f"
399 434 git config filter.blimp.smudge "blimp -s i %f"
400 435  
401   - The above git config can be done by running "blimp --init".
  436 + The above git config can be done by running "blimp init" just like git.
402 437  
403 438 When the above is done (per clone) git will automatically run blimp deflate
404 439 just before committing and blimp inflate when operations are done.
... ... @@ -406,9 +441,10 @@ let help = &quot;&quot;&quot;
406 441 This means that if you clone a git repository that already has a .gitattributes
407 442 file in it that uses the blimp filter, then you should do:
408 443  
409   - blimp --init inflate --filter
  444 + blimp init
  445 + blimp inflate --filter
410 446  
411   - This will configure the blimp filter and also find and inflate all deflated
  447 + This will configure the blimp filter and then find and inflate all deflated
412 448 files throughout the clone.
413 449 """
414 450  
... ... @@ -425,6 +461,7 @@ verbose = args[&quot;verbose&quot;].asBool
425 461 stdio = args["stdio"].asBool
426 462 onAllDeflated = args["all"].asBool
427 463 onAllFiltered = args["filter"].asBool
  464 +area = args["area"].asString
428 465  
429 466 # Can't do verbose with -s, that messes up stdout,
430 467 # read in all of stdin once and for all
... ... @@ -436,19 +473,23 @@ if stdio:
436 473 except:
437 474 quit("Failed reading stdin", 1)
438 475  
439   -
440 476 # Parse configuration files, may shadow and override each other
441 477 parseConfFile(currentDir / ".blimp.conf")
442 478 if not gitRootDir.isNil and gitRootDir != currentDir:
443 479 parseConfFile(gitRootDir / ".blimp.conf")
444 480  
  481 +if existsDir(homeDir / "blimpstore"):
  482 + parseConfFile(homeDir / "blimpstore" / ".blimp.conf")
  483 +
  484 +parseConfFile(homeDir / ".blimp.conf")
  485 +
445 486 # If we haven't gotten a blimpstore yet, we set a default one
446 487 if blimpStore.isNil:
447 488 blimpStore = homeDir / "blimpstore"
448 489  
449   -if existsDir(blimpStore):
450   - parseConfFile(blimpStore / ".blimp.conf")
451   -parseConfFile(homeDir / ".blimp.conf")
  490 +# Check if we have a configured remoteArea
  491 +if areas.hasKey(area):
  492 + remoteArea = areas[area]
452 493  
453 494 # Let's just show what we have :)
454 495 if verbose: dumpConfig()
... ... @@ -461,12 +502,8 @@ if args.showVersion: quit(&quot;blimp version: &quot; &amp; versionAsString)
461 502 if not blimpVersion.isNil and blimpVersion != versionAsString:
462 503 quit("Wrong version of blimp, configuration wants: " & blimpVersion)
463 504  
464   -# Should we install the filter?
465   -if args["init"].asBool:
466   - ensureBlimpFilter()
467   - if verbose: echo("Installed blimp filter")
468 505  
469   -let command = args["command"].asString
  506 +# Ok, let's see
470 507 var filenames = initSet[string]()
471 508  
472 509 # Add upp all files to operate on in a Set
... ... @@ -484,9 +521,16 @@ if onAllFiltered:
484 521 # Make sure the local blimpstore is setup.
485 522 setupBlimpStore()
486 523  
  524 +
487 525 # Do the deed
  526 +let command = args["command"].asString
488 527 if command != "":
489   - if command == "d" or command == "deflate":
  528 + # Should we install the filter?
  529 + if command == "init":
  530 + ensureBlimpFilter()
  531 + echo("Installed blimp filter")
  532 + quit(0)
  533 + elif command == "d" or command == "deflate":
490 534 for fn in filenames:
491 535 deflate(fn)
492 536 elif command == "i" or command == "inflate":
... ... @@ -495,12 +539,17 @@ if command != &quot;&quot;:
495 539 elif command == "remove":
496 540 for fn in filenames:
497 541 remove(fn)
  542 + elif command == "push":
  543 + for fn in filenames.items:
  544 + push(fn)
498 545 elif command == "upload":
499   - echo repr(filenames)
500 546 for fn in filenames.items:
501 547 upload(fn)
  548 + elif command == "download":
  549 + for fn in filenames.items:
  550 + download(fn)
502 551 else:
503   - quit("Unknown command: \"" & command & "\", only (d)eflate, (i)inflate, remove or upload are valid.", 6)
  552 + quit("Unknown command: \"" & command & "\", use --help for valid commands.", 6)
504 553  
505 554 # All good
506 555 quit(0)
... ...
blimp.nimble
1 1 [Package]
2 2 name = "blimp"
3   -version = "0.3"
  3 +version = "0.4"
4 4 author = "Göran Krampe"
5 5 description = "Utility that helps with big files in git, very similar to git-fat, s3annnex etc."
6 6 license = "MIT"
... ...