Blame view

blimp.nim 18.3 KB
3e2f4c14   Göran Krampe   Added upload comm...
1
  import md5, os, osproc, parseopt2, strutils, parsecfg, streams, lapp, subexes, tables, sets
74272cdb   Göran Krampe   First commit
2
  
5fc367ad   Göran Krampe   Renamed to blimp,...
3
  # blimp is a little utility program for handling large files
74272cdb   Göran Krampe   First commit
4
  # in git repositories. Its inspired by git-fat and s3annex
ba6047f5   Göran Krampe   Added areas, move...
5
6
  # but doesn't rely on S3 for storage - it uses sftp but
  # performs the file operations using externally configured commands.
aaf9bcef   Göran Krampe   Upped to version ...
7
  # It is a single binary without any dependencies.
74272cdb   Göran Krampe   First commit
8
  #
ba6047f5   Göran Krampe   Added areas, move...
9
10
11
12
  # It can also keep track of multiple remote areas and do up/downloads
  # independent from git. This is useful in build scripts, up and
  # downloading artifacts.
  #
3e2f4c14   Göran Krampe   Added upload comm...
13
  # Just run blimp --help for detailed help.
dab0c5b0   Göran Krampe   Added --version, ...
14
15
16
  
  const 
    versionMajor* = 0
ba6047f5   Göran Krampe   Added areas, move...
17
    versionMinor* = 4
386557e6   Göran Krampe   Added support for...
18
    versionPatch* = 0
dab0c5b0   Göran Krampe   Added --version, ...
19
    versionAsString* = $versionMajor & "." & $versionMinor & "." & $versionPatch
74272cdb   Göran Krampe   First commit
20
  
ba6047f5   Göran Krampe   Added areas, move...
21
22
23
24
25
26
27
28
  type
    RemoteArea = ref object of RootObj
      name*: string
      url*: string
      upload*: string
      download*: string
      delete*: string
  
b9ad52ff   Göran Krampe   Added default con...
29
  var
ba6047f5   Göran Krampe   Added areas, move...
30
    blimpStore, blimpVersion: string = nil
78569137   Göran Krampe   Added directory e...
31
    homeDir, currentDir, gitRootDir: string
ec7d9613   Göran Krampe   Added option --fi...
32
    verbose, stdio, onAllDeflated, onAllFiltered: bool
ba6047f5   Göran Krampe   Added areas, move...
33
34
35
    stdinContent, area: string = nil
    areas = newTable[string, RemoteArea]()
    remoteArea: RemoteArea
74272cdb   Göran Krampe   First commit
36
  
b9ad52ff   Göran Krampe   Added default con...
37
38
  let
    defaultConfig = """
ba6047f5   Göran Krampe   Added areas, move...
39
40
41
42
  [blimp]
  # Minimal version, otherwise stop
  # version = """ & versionAsString & """
  
aaf9bcef   Göran Krampe   Upped to version ...
43
  # Set your local blimpstore directory. You can use %home%, %cwd% and %gitroot% in paths, works cross platform.
78569137   Göran Krampe   Added directory e...
44
45
46
  # Example:
  #   # Place it inside the git clone
  #   blimpstore = "%gitroot%/.git/blimpstore"
ba6047f5   Göran Krampe   Added areas, move...
47
  #
78569137   Göran Krampe   Added directory e...
48
49
50
51
52
53
54
  #   # Place it in current working directory (not very useful)
  #   blimpstore = "%cwd%/blimpstore"
  #
  # Default:
  #   # Place it in the users home directory
  #   blimpstore = "%home%/blimpstore"
  
ba6047f5   Göran Krampe   Added areas, move...
55
56
57
58
59
60
61
  [areas]
  # This is where ypu define up/download areas. The area called "remote" is the
  # default area used unless --area is used.
  # $1 is the blimp filename, $2 is the remote url and $3 is the local blimpstore directory set above.
  remote-url = "blimpuser@some-server.com:/var/opt/blimpstore"
  remote-upload = "scp -pq $3/$1 $2/"
  remote-download = "scp -pq $2/$1 $3/"
74272cdb   Göran Krampe   First commit
62
  
ba6047f5   Göran Krampe   Added areas, move...
63
64
65
66
67
  # Example area that can be used with upload/download commands and --area option.
  release-url = "blimpuser@some-server.com:/var/opt/release"
  release-upload = "scp -r $3/$1 $2/"
  release-download = "scp -r $2/$1 $3/"
  """ 
386557e6   Göran Krampe   Added support for...
68
69
  
  proc cmd(cmd: string): string =
78569137   Göran Krampe   Added directory e...
70
    try:
ba6047f5   Göran Krampe   Added areas, move...
71
72
73
74
75
76
      # Otherwise pipes will not work for git commands etc
      when defined(windows):
        let tup = execCmdEx("cmd /c \"" & cmd & "\"")
      else:
        let tup = execCmdEx(cmd) 
      #echo "cmd: " & $cmd & "err:" & $tup[1]
78569137   Göran Krampe   Added directory e...
77
78
79
80
81
82
83
      if tup[1] == 0:
        result = strip(tup[0])
      else:
        result = nil
    except:
      result = nil
  
386557e6   Göran Krampe   Added support for...
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
  # Find git root dir or nil
  proc gitRoot(): string =
    cmd("git rev-parse --show-toplevel")
  
  # Git config
  proc gitConfigSet(key, val: string) =
    discard cmd("git config " & $key & " " & $val)
  
  proc gitConfigGet(key: string): string =
    cmd("git config --get " & $key)
  
  # Set blimp filter
  proc setBlimpFilter() =
    gitConfigSet("filter.blimp.clean", "\"blimp -s d %f\"")
    gitConfigSet("filter.blimp.smudge", "\"blimp -s i %f\"")
  
  
  # Ensure the filter is set
  proc ensureBlimpFilter() =
    if gitConfigGet("filter.blimp.clean") != "":
      setBlimpFilter()
  
  
78569137   Göran Krampe   Added directory e...
107
108
109
110
111
112
113
114
  # Simple expansion of %home%, %cwd% and %gitroot%
  proc expandDirs(templ: string): string =
    result = templ.replace("%home%", homeDir)
    result = result.replace("%cwd%", currentDir)
    if result.contains("%gitroot%"):
      if gitRootDir.isNil: quit("Not in a git clone, can not expand %gitroot% in '" & templ & "'") 
      result = result.replace("%gitroot%", gitRootDir)
  
dab0c5b0   Göran Krampe   Added --version, ...
115
  # Load a blimp.conf file
74272cdb   Göran Krampe   First commit
116
  proc parseConfFile(filename: string) =
74272cdb   Göran Krampe   First commit
117
118
    var f = newFileStream(filename, fmRead)
    if f != nil:
dab0c5b0   Göran Krampe   Added --version, ...
119
      if verbose: echo "Reading config: " & filename
386557e6   Göran Krampe   Added support for...
120
      var p: CfgParser  
74272cdb   Göran Krampe   First commit
121
      open(p, f, filename)
ba6047f5   Göran Krampe   Added areas, move...
122
123
      var section: string
      var area: RemoteArea
74272cdb   Göran Krampe   First commit
124
125
126
127
128
129
      while true:
        var e = next(p)
        case e.kind
        of cfgEof: 
          break
        of cfgSectionStart:
ba6047f5   Göran Krampe   Added areas, move...
130
          section = e.section
74272cdb   Göran Krampe   First commit
131
        of cfgKeyValuePair:
ba6047f5   Göran Krampe   Added areas, move...
132
133
134
135
136
137
138
139
140
          case section
          of "blimp":
            case e.key
            of "blimpstore":
              if blimpStore.isNil: blimpStore = e.value                  
            of "version":
              if blimpVersion.isNil: blimpVersion = e.value
            else:
              quit("Unknown blimp configuration: " & e.key)
74272cdb   Göran Krampe   First commit
141
          else:
ba6047f5   Göran Krampe   Added areas, move...
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
            # Then we presume its an area
            if area.isNil or area.name != section:
              area = RemoteArea(name: section)
              areas[area.name] = area
            case e.key
            of "url":
              if area.url.isNil: area.url = expandDirs(e.value)
            of "upload":
              if area.upload.isNil: area.upload = expandDirs(e.value)
            of "download":
              if area.download.isNil: area.download = expandDirs(e.value)
            of "delete":
              if area.delete.isNil: area.delete = expandDirs(e.value)
            else:
              quit("Unknown area configuration: " & e.key)
74272cdb   Göran Krampe   First commit
157
158
159
160
161
162
        of cfgOption:
          quit("Unknown configuration: " & e.key)
        of cfgError:
          quit("Parsing " & filename & ": " & e.msg)
      close(p)
  
db95b901   Göran Krampe   Various fixes, mo...
163
  # Trivial helper to enable verbose
ba6047f5   Göran Krampe   Added areas, move...
164
  proc run(cmd: string): int =
db95b901   Göran Krampe   Various fixes, mo...
165
166
    if verbose: echo(cmd)
    execCmd(cmd)
74272cdb   Göran Krampe   First commit
167
  
ba6047f5   Göran Krampe   Added areas, move...
168
169
170
171
172
  
  # Upload a file to the remote area
  proc uploadFile(filename, fromDir: string) =
    if remoteArea.isNil:
      echo("Remote area not set in configuration file, skipping uploading content:\n\t" & filename)
74272cdb   Göran Krampe   First commit
173
      return
ba6047f5   Göran Krampe   Added areas, move...
174
    let errorCode = run(format(remoteArea.upload, filename, remoteArea.url, fromDir))
b9ad52ff   Göran Krampe   Added default con...
175
    if errorCode != 0:
ba6047f5   Göran Krampe   Added areas, move...
176
      quit("Something went wrong uploading " & filename & " to " & remoteArea.url, 2)
74272cdb   Göran Krampe   First commit
177
    
ba6047f5   Göran Krampe   Added areas, move...
178
179
180
181
182
  # Download a file to the remote area
  proc downloadFile(filename, toDir: string) =
    if remoteArea.isNil:
      quit("Remote area not set in configuration file, can not download content:\n\t" & filename)
    let errorCode = run(format(remoteArea.download, filename, remoteArea.url, toDir))
b9ad52ff   Göran Krampe   Added default con...
183
    if errorCode != 0:
ba6047f5   Göran Krampe   Added areas, move...
184
      quit("Something went wrong downloading " & filename & " from " & remoteArea.url, 3)
5fc367ad   Göran Krampe   Renamed to blimp,...
185
  
e914743a   Göran Krampe   Added remove comm...
186
  # Delete a file from the remote master blimpStore
ba6047f5   Göran Krampe   Added areas, move...
187
188
  proc remoteDeleteFile(filename: string) =
    if remoteArea.isNil:
e914743a   Göran Krampe   Added remove comm...
189
      return
ba6047f5   Göran Krampe   Added areas, move...
190
    let errorCode = run(format(remoteArea.delete, filename, remoteArea.url, blimpStore))
e914743a   Göran Krampe   Added remove comm...
191
    if errorCode != 0:
ba6047f5   Göran Krampe   Added areas, move...
192
      quit("Something went wrong deleting " & filename & " from " & remoteArea.url, 3)
5fc367ad   Göran Krampe   Renamed to blimp,...
193
  
3e2f4c14   Göran Krampe   Added upload comm...
194
  # Copy content to blimpStore and upload if it was a new file or upload == true.
5fc367ad   Göran Krampe   Renamed to blimp,...
195
196
  proc copyToBlimpStore(filename, blimpFilename: string) =
    if not existsFile(blimpStore / blimpFilename):
aaf9bcef   Göran Krampe   Upped to version ...
197
198
199
200
201
202
203
      if stdio:
        try:
          writeFile(blimpStore / blimpFilename, stdinContent)
        except:
          quit("Failed writing file: " & blimpStore / blimpFilename & " from stdin", 1)
      else:
        copyFile(filename, blimpStore / blimpFilename)
ba6047f5   Göran Krampe   Added areas, move...
204
      uploadFile(blimpFilename, blimpStore)
3e2f4c14   Göran Krampe   Added upload comm...
205
    
5fc367ad   Göran Krampe   Renamed to blimp,...
206
207
  
  # Copy content from blimpStore, and downloading first if needed
e914743a   Göran Krampe   Added remove comm...
208
  proc copyFromBlimpStore(blimpFilename, filename: string) =
5fc367ad   Göran Krampe   Renamed to blimp,...
209
    if not existsFile(blimpStore / blimpFilename):
ba6047f5   Göran Krampe   Added areas, move...
210
      downloadFile(blimpFilename, blimpStore)
aaf9bcef   Göran Krampe   Upped to version ...
211
212
213
214
215
216
217
218
    if stdio:
      try:
        var content = readFile(blimpStore / blimpFilename)
        write(stdout, content)
      except:
        quit("Failed reading file: " & blimpStore / blimpFilename & " to stdout", 1)
    else:
      copyFile(blimpStore / blimpFilename, filename)
74272cdb   Göran Krampe   First commit
219
  
e914743a   Göran Krampe   Added remove comm...
220
221
222
223
224
225
  # Delete from blimpStore and remote.
  proc deleteFromBlimpStore(blimpFilename, filename: string) =
    if existsFile(blimpStore / blimpFilename):
      removeFile(blimpStore / blimpFilename)
    remoteDeleteFile(blimpFilename)
  
aaf9bcef   Göran Krampe   Upped to version ...
226
  proc blimpFileNameFromString(line: string): string =
386557e6   Göran Krampe   Added support for...
227
228
    let hashline = split(strip(line), ':')
    if hashline[0] == "blimphash":
e914743a   Göran Krampe   Added remove comm...
229
230
231
232
      result = hashline[1]
    else:
      result = nil
  
3e2f4c14   Göran Krampe   Added upload comm...
233
  # Pick out blimpFilename (filename & "-" & hash) or nil
aaf9bcef   Göran Krampe   Upped to version ...
234
235
236
237
238
239
240
241
242
  proc blimpFileName(filename: string): string =
    if stdio:
      blimpFileNameFromString(stdinContent)
    else:
      var hashfile: File
      if not open(hashfile, filename):
        quit("Failed opening file: " & filename, 4)
      blimpFileNameFromString(string(readLine(hashfile)))  
  
e914743a   Göran Krampe   Added remove comm...
243
244
  # Get hash and compute blimpFilename
  proc computeBlimpFilename(filename: string): string =
dab0c5b0   Göran Krampe   Added --version, ...
245
246
247
248
249
250
    var content: string
    try:
      content = readFile(filename)
    except:
      quit("Failed opening file: " & filename, 1)
    let hash = getMD5(content)
14d16aa7   Göran Krampe   Now the git filte...
251
    result = extractFilename(filename) & "-" & hash
e914743a   Göran Krampe   Added remove comm...
252
   
5fc367ad   Göran Krampe   Renamed to blimp,...
253
  # Copy original file to blimpStore and replace with hash stub in git.
74272cdb   Göran Krampe   First commit
254
  proc deflate(filename: string) =
dab0c5b0   Göran Krampe   Added --version, ...
255
256
257
258
259
260
261
    if verbose: echo "Deflating " & filename
    var blimpFilename = blimpFilename(filename)
    if not blimpFilename.isNil:
      echo("\t" & filename & " is already deflated, skipping.")
    else:
      blimpFilename = computeBlimpFilename(filename)
      copyToBlimpStore(filename, blimpFilename)
aaf9bcef   Göran Krampe   Upped to version ...
262
      if stdio:
386557e6   Göran Krampe   Added support for...
263
        write(stdout, "blimphash:" & blimpFilename)
aaf9bcef   Göran Krampe   Upped to version ...
264
      else:
386557e6   Göran Krampe   Added support for...
265
        writeFile(filename, "blimphash:" & blimpFilename)
dab0c5b0   Göran Krampe   Added --version, ...
266
      if verbose: echo("\t" & filename & " deflated.")
e914743a   Göran Krampe   Added remove comm...
267
  
386557e6   Göran Krampe   Added support for...
268
269
  # Iterator over all deflated files in the git clone
  iterator allDeflated() =
7e507b62   Göran Krampe   Fixes discovered ...
270
    let filenames = cmd("git ls-files " & gitRootDir).split({'\l', '\c'})
386557e6   Göran Krampe   Added support for...
271
272
273
    for fn in filenames:
      if not blimpFilename(fn).isNil:
        yield fn
ec7d9613   Göran Krampe   Added option --fi...
274
275
276
        
  # Iterator over all files matching the blimp filter in the git clone
  iterator allFiltered() =
7e507b62   Göran Krampe   Fixes discovered ...
277
    let lines = cmd("git ls-files | git check-attr --stdin filter").split({'\l', '\c'})
ec7d9613   Göran Krampe   Added option --fi...
278
279
280
281
    for line in lines:
      let status = line.split(':')
      if strip(status[2]) == "blimp":
        yield status[0]
386557e6   Göran Krampe   Added support for...
282
  
5fc367ad   Göran Krampe   Renamed to blimp,...
283
  # Parse out hash from hash stub and copy back original content from blimpStore.
74272cdb   Göran Krampe   First commit
284
  proc inflate(filename: string) =
dab0c5b0   Göran Krampe   Added --version, ...
285
    if verbose: echo "Inflating " & filename
e914743a   Göran Krampe   Added remove comm...
286
287
    let blimpFilename = blimpFilename(filename)
    if blimpFilename.isNil:
dab0c5b0   Göran Krampe   Added --version, ...
288
      echo("\t" & filename & " is not deflated, skipping.")
e914743a   Göran Krampe   Added remove comm...
289
290
    else:
      copyFromBlimpStore(blimpfilename, filename)
dab0c5b0   Göran Krampe   Added --version, ...
291
      if verbose: echo("\t" & filename & " inflated.")
e914743a   Göran Krampe   Added remove comm...
292
293
294
295
296
297
298
  
  # Inflates file first (if deflated) and then removes current content for it,
  # both locally and in remote.
  proc remove(filename: string) =
    var blimpFilename = blimpFilename(filename)
    if not blimpFilename.isNil:
      copyFromBlimpStore(blimpfilename, filename)
74272cdb   Göran Krampe   First commit
299
    else:
e914743a   Göran Krampe   Added remove comm...
300
301
302
      blimpFilename = computeBlimpFilename(filename)
    deleteFromBlimpStore(blimpfilename, filename)
    echo("\t" & filename & " content removed from blimpstore locally and remotely.")
5fc367ad   Göran Krampe   Renamed to blimp,...
303
  
dab0c5b0   Göran Krampe   Added --version, ...
304
  
ba6047f5   Göran Krampe   Added areas, move...
305
306
  # Make sure a file already in blimpstore is uploaded to remote area.
  proc push(filename: string) =
3e2f4c14   Göran Krampe   Added upload comm...
307
308
309
310
311
    if verbose: echo "Uploading " & filename
    var blimpFilename = blimpFilename(filename)
    if blimpFilename.isNil:
      blimpFilename = computeBlimpFilename(filename)
    if existsFile(blimpStore / blimpFilename):
ba6047f5   Göran Krampe   Added areas, move...
312
      uploadFile(blimpFilename, blimpStore)
3e2f4c14   Göran Krampe   Added upload comm...
313
314
315
316
      if verbose: echo("\t" & filename & " uploaded.")
    else:
      if verbose: echo("\t" & filename & " is not in blimpstore, skipping.")
  
ba6047f5   Göran Krampe   Added areas, move...
317
318
319
320
321
322
323
324
325
326
327
  # Upload a file to a remote area
  proc upload(filename: string) =
    if verbose: echo "Uploading " & filename
    uploadFile(filename, currentDir)
    if verbose: echo("\t" & filename & " uploaded.")
      
  # Download a file from a remote area.
  proc download(filename: string) =
    if verbose: echo "Downloading " & filename
    downloadFile(filename, currentDir)
    if verbose: echo("\t" & filename & " downloaded.")
3e2f4c14   Göran Krampe   Added upload comm...
328
  
dab0c5b0   Göran Krampe   Added --version, ...
329
330
331
332
333
334
335
336
337
338
339
340
341
  proc setupBlimpStore() =
    try:
      if not existsDir(blimpStore):
        createDir(blimpStore)
    except:
      quit("Could not create " & blimpStore & " directory.", 1)
  
    try:
      if not existsFile(blimpStore / ".blimp.conf"):
        writeFile(blimpStore / ".blimp.conf", defaultConfig)
    except:
      quit("Could not create .blimp.conf config file in " & blimpStore & " directory.", 1)
  
78569137   Göran Krampe   Added directory e...
342
343
344
  proc `$`(x: string): string =
    if x.isNil: "nil" else: x
  
dab0c5b0   Göran Krampe   Added --version, ...
345
346
347
  proc dumpConfig() =
    echo "\nDump of configuration:"
    echo "\tblimpStore: " & blimpStore
ba6047f5   Göran Krampe   Added areas, move...
348
349
350
    echo "\tremote-url: " & remoteArea.url
    echo "\tremote-upload: " & remoteArea.upload
    echo "\tremote-download: " & remoteArea.download
aaf9bcef   Göran Krampe   Upped to version ...
351
    echo "\tblimpVersion: " & $blimpVersion
dab0c5b0   Göran Krampe   Added --version, ...
352
    echo "\n"
74272cdb   Göran Krampe   First commit
353
  
386557e6   Göran Krampe   Added support for...
354
  let synopsis = """
db95b901   Göran Krampe   Various fixes, mo...
355
    blimp [options] <command> <filenames...>
ba6047f5   Göran Krampe   Added areas, move...
356
357
358
      -h,--help                  Show this
      --version                  Show version of blimp
      -v,--verbose               Verbosity, only works without -s
386557e6   Göran Krampe   Added support for...
359
      ----------
ba6047f5   Göran Krampe   Added areas, move...
360
361
362
363
      <command>  (string)        (d)eflate, (i)nflate, init, remove, push, upload, download
      --all                      Operate on all deflated files in clone
      -f,--filter                Operate on all files matching blimp filter
      -a,--area (default remote) The area to use for remote up/downloads
386557e6   Göran Krampe   Added support for...
364
      ----------
ba6047f5   Göran Krampe   Added areas, move...
365
366
      -s,--stdio                 If given, use stdin/stdout for content.
      <filenames> (string...)    One or more filepaths to inflate/deflate
386557e6   Göran Krampe   Added support for...
367
368
  """
  let help = """
3e2f4c14   Göran Krampe   Added upload comm...
369
  
aaf9bcef   Göran Krampe   Upped to version ...
370
371
372
    blimp is a little utility program for handling large files
    in git repositories. Its inspired by git-fat and s3annex
    but doesn't rely on S3 for storage - it uses rsync like git-fat.
386557e6   Göran Krampe   Added support for...
373
374
    It is a single binary without any dependencies. Its not as advanced
    as git-fat but basically does the same thing.
aaf9bcef   Göran Krampe   Upped to version ...
375
376
377
  
    Manual use:
  
386557e6   Göran Krampe   Added support for...
378
379
    Use "blimp d mybigfile" to deflate a file, typically before commit.
    Use "blimp i mybigfile" to inflate it back to original content.
aaf9bcef   Göran Krampe   Upped to version ...
380
  
3e2f4c14   Göran Krampe   Added upload comm...
381
382
    When deflated the file only has this content:
    
386557e6   Göran Krampe   Added support for...
383
      "blimphash:" <filename> "-" <md5sum>
aaf9bcef   Göran Krampe   Upped to version ...
384
  
386557e6   Göran Krampe   Added support for...
385
    Deflate also copies the real content to your local blimpstore:
aaf9bcef   Göran Krampe   Upped to version ...
386
    
386557e6   Göran Krampe   Added support for...
387
      <blimpstore>/<filename>-<md5sum>
aaf9bcef   Göran Krampe   Upped to version ...
388
  
386557e6   Göran Krampe   Added support for...
389
    ...and if configured also uploads it to "remote", using rsync.
aaf9bcef   Göran Krampe   Upped to version ...
390
391
392
393
  
    Configuration is in these locations in order:
   
      ./.blimp.conf
386557e6   Göran Krampe   Added support for...
394
      <gitroot>/.blimp.conf
aaf9bcef   Göran Krampe   Upped to version ...
395
396
397
398
399
400
401
402
403
404
405
406
      ~/<blimpstore>/.blimp.conf
      ~/.blimp.conf
  
    This way you can have settings per directory, per git clone,
    per store and per user. A default blimpstore with a commented .blimp.conf
    is created in ~/blimpstore if you run blimp and no .blimp.conf is found.
  
    Edit ~/blimpstore/.blimp.conf (or in another location) and set a proper
    remote and the proper rsync password. This ensures that its also properly
    synced with a master rsync repository that is typically shared.
  
    Inflate will bring back the original content by copying from
386557e6   Göran Krampe   Added support for...
407
    your local blimpstore, and if its not there, first download from the remote.
aaf9bcef   Göran Krampe   Upped to version ...
408
409
410
411
412
    Use this whenever you need to work/edit the big file - in order to get
    its real content.
  
    The filenames given are all processed. If -s is used content is processed via
    stdin/stdout and only one filename can be passed. This is used when running blimp
386557e6   Göran Krampe   Added support for...
413
    via a git filter (smudge/clean), see below.
aaf9bcef   Göran Krampe   Upped to version ...
414
415
416
417
418
  
    The remove command (no single character shortcut) will remove the file(s) content
    both from the local blimpstore and from the remote. This only removes
    the current content version, not older versions. The file itself is first
    inflated, if needed, and not deleted. This only "unblimps" the file.
386557e6   Göran Krampe   Added support for...
419
    
ba6047f5   Göran Krampe   Added areas, move...
420
    The push command (no single character shortcut) will force upload the given file
3e2f4c14   Göran Krampe   Added upload comm...
421
    from the local blimpstore to the remote. This is normally done automatically,
ba6047f5   Göran Krampe   Added areas, move...
422
423
424
425
    but this way you can make sure they are synced onto the remote.
    
    The upload and download commands are used to distribute artifacts typically in a
    build script. If no --area is given, they use the standard "remote" area.
3e2f4c14   Göran Krampe   Added upload comm...
426
    
386557e6   Göran Krampe   Added support for...
427
428
429
    In order to have blimp work automatically you can:
    
    * Create a .gitattributes file with lines like:
ec7d9613   Göran Krampe   Added option --fi...
430
      *.png filter=blimp binary
386557e6   Göran Krampe   Added support for...
431
432
433
434
435
      
    * Configure blimp as a filter by running:
      git config filter.blimp.clean "blimp -s d %f"
      git config filter.blimp.smudge "blimp -s i %f"
  
ba6047f5   Göran Krampe   Added areas, move...
436
    The above git config can be done by running "blimp init" just like git.
386557e6   Göran Krampe   Added support for...
437
438
439
440
441
442
443
  
    When the above is done (per clone) git will automatically run blimp deflate
    just before committing and blimp inflate when operations are done.
    
    This means that if you clone a git repository that already has a .gitattributes
    file in it that uses the blimp filter, then you should do:
    
ba6047f5   Göran Krampe   Added areas, move...
444
445
      blimp init
      blimp inflate --filter
386557e6   Göran Krampe   Added support for...
446
    
ba6047f5   Göran Krampe   Added areas, move...
447
    This will configure the blimp filter and then find and inflate all deflated
386557e6   Göran Krampe   Added support for...
448
    files throughout the clone.
aaf9bcef   Göran Krampe   Upped to version ...
449
  """
db95b901   Göran Krampe   Various fixes, mo...
450
  
74272cdb   Göran Krampe   First commit
451
  ################################ main #####################################
78569137   Göran Krampe   Added directory e...
452
453
454
455
456
  # Set some dirs
  homeDir = getHomeDir()
  homeDir = homeDir[0.. -2] # Not sure why it keeps a trailing "/" on Linux
  currentDir = getCurrentDir()
  gitRootDir = gitRoot()
7e507b62   Göran Krampe   Fixes discovered ...
457
  echo gitRootDir
74272cdb   Göran Krampe   First commit
458
  
dab0c5b0   Göran Krampe   Added --version, ...
459
  # Using lapp to get args, on parsing failure this will show usage automatically
386557e6   Göran Krampe   Added support for...
460
  var args = parse(synopsis)
dab0c5b0   Göran Krampe   Added --version, ...
461
  verbose = args["verbose"].asBool
aaf9bcef   Göran Krampe   Upped to version ...
462
  stdio = args["stdio"].asBool
ec7d9613   Göran Krampe   Added option --fi...
463
464
  onAllDeflated = args["all"].asBool
  onAllFiltered = args["filter"].asBool
ba6047f5   Göran Krampe   Added areas, move...
465
  area = args["area"].asString
aaf9bcef   Göran Krampe   Upped to version ...
466
467
468
469
470
471
472
473
474
475
476
  
  # Can't do verbose with -s, that messes up stdout,
  # read in all of stdin once and for all
  if stdio:
    verbose = false
    try:
      stdinContent = readAll(stdin)
      close(stdin)
    except:
      quit("Failed reading stdin", 1)
  
dab0c5b0   Göran Krampe   Added --version, ...
477
  # Parse configuration files, may shadow and override each other
78569137   Göran Krampe   Added directory e...
478
  parseConfFile(currentDir / ".blimp.conf")
386557e6   Göran Krampe   Added support for...
479
  if not gitRootDir.isNil and gitRootDir != currentDir:
78569137   Göran Krampe   Added directory e...
480
    parseConfFile(gitRootDir / ".blimp.conf")
74272cdb   Göran Krampe   First commit
481
  
ba6047f5   Göran Krampe   Added areas, move...
482
483
484
485
486
  if existsDir(homeDir / "blimpstore"):
    parseConfFile(homeDir / "blimpstore" / ".blimp.conf")
  
  parseConfFile(homeDir / ".blimp.conf")
  
dab0c5b0   Göran Krampe   Added --version, ...
487
488
  # If we haven't gotten a blimpstore yet, we set a default one
  if blimpStore.isNil:
78569137   Göran Krampe   Added directory e...
489
    blimpStore = homeDir / "blimpstore"
74272cdb   Göran Krampe   First commit
490
  
ba6047f5   Göran Krampe   Added areas, move...
491
492
493
  # Check if we have a configured remoteArea
  if areas.hasKey(area):
    remoteArea = areas[area]
74272cdb   Göran Krampe   First commit
494
  
386557e6   Göran Krampe   Added support for...
495
  # Let's just show what we have :)
dab0c5b0   Göran Krampe   Added --version, ...
496
  if verbose: dumpConfig()
e914743a   Göran Krampe   Added remove comm...
497
  
dab0c5b0   Göran Krampe   Added --version, ...
498
  # These two are special, they short out
386557e6   Göran Krampe   Added support for...
499
  if args.showHelp: quit(synopsis & help)
dab0c5b0   Göran Krampe   Added --version, ...
500
  if args.showVersion: quit("blimp version: " & versionAsString)
74272cdb   Göran Krampe   First commit
501
  
aaf9bcef   Göran Krampe   Upped to version ...
502
503
504
505
  # Check blimpVersion
  if not blimpVersion.isNil and blimpVersion != versionAsString:
    quit("Wrong version of blimp, configuration wants: " & blimpVersion)
  
386557e6   Göran Krampe   Added support for...
506
  
ba6047f5   Göran Krampe   Added areas, move...
507
  # Ok, let's see 
3e2f4c14   Göran Krampe   Added upload comm...
508
509
510
  var filenames = initSet[string]()
  
  # Add upp all files to operate on in a Set
386557e6   Göran Krampe   Added support for...
511
  if args.hasKey("filenames"):
3e2f4c14   Göran Krampe   Added upload comm...
512
513
514
515
516
517
518
519
520
    for f in args["filenames"].asSeq:
      if f.asString != "":
        filenames.incl(f.asString)
  if onAllDeflated:
    for fn in allDeflated():
      filenames.incl(fn)
  if onAllFiltered:
    for fn in allFiltered():
      filenames.incl(fn)
dab0c5b0   Göran Krampe   Added --version, ...
521
522
523
524
  
  # Make sure the local blimpstore is setup.
  setupBlimpStore()
  
ba6047f5   Göran Krampe   Added areas, move...
525
  
74272cdb   Göran Krampe   First commit
526
  # Do the deed
ba6047f5   Göran Krampe   Added areas, move...
527
  let command = args["command"].asString
386557e6   Göran Krampe   Added support for...
528
  if command != "":
ba6047f5   Göran Krampe   Added areas, move...
529
530
531
532
533
534
    # Should we install the filter?
    if command == "init":
      ensureBlimpFilter()
      echo("Installed blimp filter")
      quit(0)
    elif command == "d" or command == "deflate":
3e2f4c14   Göran Krampe   Added upload comm...
535
536
      for fn in filenames:
        deflate(fn)
386557e6   Göran Krampe   Added support for...
537
    elif command == "i" or command == "inflate":
3e2f4c14   Göran Krampe   Added upload comm...
538
539
      for fn in filenames:
        inflate(fn)
386557e6   Göran Krampe   Added support for...
540
541
    elif command == "remove":
      for fn in filenames:
3e2f4c14   Göran Krampe   Added upload comm...
542
        remove(fn)
ba6047f5   Göran Krampe   Added areas, move...
543
544
545
    elif command == "push":
      for fn in filenames.items:
        push(fn)
3e2f4c14   Göran Krampe   Added upload comm...
546
    elif command == "upload":
3e2f4c14   Göran Krampe   Added upload comm...
547
548
      for fn in filenames.items:
        upload(fn)
ba6047f5   Göran Krampe   Added areas, move...
549
550
551
    elif command == "download":
      for fn in filenames.items:
        download(fn)
386557e6   Göran Krampe   Added support for...
552
    else:
ba6047f5   Göran Krampe   Added areas, move...
553
      quit("Unknown command: \"" & command & "\", use --help for valid commands.", 6)
74272cdb   Göran Krampe   First commit
554
555
556
  
  # All good
  quit(0)