Blame view

blimp.nim 19 KB
3e2f4c14   Göran Krampe   Added upload comm...
1
  import md5, os, osproc, parseopt2, strutils, parsecfg, streams, lapp, subexes, tables, sets
74272cdb   Göran Krampe   First commit
2
  
5fc367ad   Göran Krampe   Renamed to blimp,...
3
  # blimp is a little utility program for handling large files
74272cdb   Göran Krampe   First commit
4
  # in git repositories. Its inspired by git-fat and s3annex
6884d534   Göran Krampe   Fixed unixy path ...
5
  # but doesn't rely on S3 for storage - it uses scp but
ba6047f5   Göran Krampe   Added areas, move...
6
  # performs the file operations using externally configured commands.
aaf9bcef   Göran Krampe   Upped to version ...
7
  # It is a single binary without any dependencies.
74272cdb   Göran Krampe   First commit
8
  #
ba6047f5   Göran Krampe   Added areas, move...
9
10
11
12
  # It can also keep track of multiple remote areas and do up/downloads
  # independent from git. This is useful in build scripts, up and
  # downloading artifacts.
  #
3e2f4c14   Göran Krampe   Added upload comm...
13
  # Just run blimp --help for detailed help.
dab0c5b0   Göran Krampe   Added --version, ...
14
15
16
  
  const 
    versionMajor* = 0
ba6047f5   Göran Krampe   Added areas, move...
17
    versionMinor* = 4
386557e6   Göran Krampe   Added support for...
18
    versionPatch* = 0
dab0c5b0   Göran Krampe   Added --version, ...
19
    versionAsString* = $versionMajor & "." & $versionMinor & "." & $versionPatch
74272cdb   Göran Krampe   First commit
20
  
ba6047f5   Göran Krampe   Added areas, move...
21
22
23
24
25
26
27
28
  type
    RemoteArea = ref object of RootObj
      name*: string
      url*: string
      upload*: string
      download*: string
      delete*: string
  
b9ad52ff   Göran Krampe   Added default con...
29
  var
ba6047f5   Göran Krampe   Added areas, move...
30
    blimpStore, blimpVersion: string = nil
78569137   Göran Krampe   Added directory e...
31
    homeDir, currentDir, gitRootDir: string
ec7d9613   Göran Krampe   Added option --fi...
32
    verbose, stdio, onAllDeflated, onAllFiltered: bool
ba6047f5   Göran Krampe   Added areas, move...
33
34
35
    stdinContent, area: string = nil
    areas = newTable[string, RemoteArea]()
    remoteArea: RemoteArea
74272cdb   Göran Krampe   First commit
36
  
6884d534   Göran Krampe   Fixed unixy path ...
37
  let defaultConfig = """
ba6047f5   Göran Krampe   Added areas, move...
38
39
  [blimp]
  # Minimal version, otherwise stop
7d1929db   Göran Krampe   Added version in ...
40
  version = """ & versionAsString & """
6884d534   Göran Krampe   Fixed unixy path ...
41
42
43
44
  #
  # Set your local blimpstore directory. You can use %home%,
  # %cwd% and %gitroot% in paths, works cross platform.
  #
78569137   Göran Krampe   Added directory e...
45
46
47
  # Example:
  #   # Place it inside the git clone
  #   blimpstore = "%gitroot%/.git/blimpstore"
ba6047f5   Göran Krampe   Added areas, move...
48
  #
78569137   Göran Krampe   Added directory e...
49
50
51
52
53
  #   # Place it in current working directory (not very useful)
  #   blimpstore = "%cwd%/blimpstore"
  #
  # Default:
  #   # Place it in the users home directory
6884d534   Göran Krampe   Fixed unixy path ...
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
  blimpstore = "%home%/blimpstore"
  
  
  [remote]
  # The "remote" area represents the shared remote blimpstore.
  # This area is the default area used unless --area is used.
  url = "blimp@build.3dicc.com:/var/opt/blimpstore"
  
  # On Windows we rely on msysgit that includes scp.
  # On XP, make sure to put your ssh keys
  # in c:\Program Files\git\.ssh (slightly odd place)
  # On Win7 it uses c:\Users\gokr\.ssh - you can check with
  # "ssh -v somehost" to see where it looks for the keys.
  #
  # When expanding the command templates below:
  #   $1 is the blimp filename
  #   $2 is the area url above
  #   $3 is the local blimpstore directory
eb67d6ae   Göran Krampe   Added compression...
72
73
  upload = "scp -pqC '$3/$1' '$2/'"
  download = "scp -pqC '$2/$1' '$3/'"
6884d534   Göran Krampe   Fixed unixy path ...
74
75
76
77
78
79
  
  
  [release]
  # This area can be used with upload/download commands and --area option.
  # -r makes it possible to give directories as arguments for recursive handling.
  url = "blimp@build.3dicc.com:/var/opt/release"
eb67d6ae   Göran Krampe   Added compression...
80
81
  upload = "scp -rC '$3/$1' '$2/'"
  download = "scp -rC '$2/$1' '$3/'"
6884d534   Göran Krampe   Fixed unixy path ...
82
83
  """
  
386557e6   Göran Krampe   Added support for...
84
85
  
  proc cmd(cmd: string): string =
78569137   Göran Krampe   Added directory e...
86
    try:
ba6047f5   Göran Krampe   Added areas, move...
87
88
89
90
91
92
      # Otherwise pipes will not work for git commands etc
      when defined(windows):
        let tup = execCmdEx("cmd /c \"" & cmd & "\"")
      else:
        let tup = execCmdEx(cmd) 
      #echo "cmd: " & $cmd & "err:" & $tup[1]
78569137   Göran Krampe   Added directory e...
93
94
95
96
97
98
99
      if tup[1] == 0:
        result = strip(tup[0])
      else:
        result = nil
    except:
      result = nil
  
386557e6   Göran Krampe   Added support for...
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
  # Find git root dir or nil
  proc gitRoot(): string =
    cmd("git rev-parse --show-toplevel")
  
  # Git config
  proc gitConfigSet(key, val: string) =
    discard cmd("git config " & $key & " " & $val)
  
  proc gitConfigGet(key: string): string =
    cmd("git config --get " & $key)
  
  # Set blimp filter
  proc setBlimpFilter() =
    gitConfigSet("filter.blimp.clean", "\"blimp -s d %f\"")
    gitConfigSet("filter.blimp.smudge", "\"blimp -s i %f\"")
  
  
  # Ensure the filter is set
  proc ensureBlimpFilter() =
    if gitConfigGet("filter.blimp.clean") != "":
      setBlimpFilter()
  
  
78569137   Göran Krampe   Added directory e...
123
124
125
126
127
128
129
130
  # Simple expansion of %home%, %cwd% and %gitroot%
  proc expandDirs(templ: string): string =
    result = templ.replace("%home%", homeDir)
    result = result.replace("%cwd%", currentDir)
    if result.contains("%gitroot%"):
      if gitRootDir.isNil: quit("Not in a git clone, can not expand %gitroot% in '" & templ & "'") 
      result = result.replace("%gitroot%", gitRootDir)
  
dab0c5b0   Göran Krampe   Added --version, ...
131
  # Load a blimp.conf file
74272cdb   Göran Krampe   First commit
132
  proc parseConfFile(filename: string) =
74272cdb   Göran Krampe   First commit
133
134
    var f = newFileStream(filename, fmRead)
    if f != nil:
dab0c5b0   Göran Krampe   Added --version, ...
135
      if verbose: echo "Reading config: " & filename
386557e6   Göran Krampe   Added support for...
136
      var p: CfgParser  
74272cdb   Göran Krampe   First commit
137
      open(p, f, filename)
ba6047f5   Göran Krampe   Added areas, move...
138
139
      var section: string
      var area: RemoteArea
74272cdb   Göran Krampe   First commit
140
141
142
143
144
145
      while true:
        var e = next(p)
        case e.kind
        of cfgEof: 
          break
        of cfgSectionStart:
ba6047f5   Göran Krampe   Added areas, move...
146
          section = e.section
74272cdb   Göran Krampe   First commit
147
        of cfgKeyValuePair:
ba6047f5   Göran Krampe   Added areas, move...
148
149
150
151
          case section
          of "blimp":
            case e.key
            of "blimpstore":
6884d534   Göran Krampe   Fixed unixy path ...
152
              if blimpStore.isNil: blimpStore = expandDirs(e.value)                
ba6047f5   Göran Krampe   Added areas, move...
153
154
155
156
            of "version":
              if blimpVersion.isNil: blimpVersion = e.value
            else:
              quit("Unknown blimp configuration: " & e.key)
74272cdb   Göran Krampe   First commit
157
          else:
ba6047f5   Göran Krampe   Added areas, move...
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
            # Then we presume its an area
            if area.isNil or area.name != section:
              area = RemoteArea(name: section)
              areas[area.name] = area
            case e.key
            of "url":
              if area.url.isNil: area.url = expandDirs(e.value)
            of "upload":
              if area.upload.isNil: area.upload = expandDirs(e.value)
            of "download":
              if area.download.isNil: area.download = expandDirs(e.value)
            of "delete":
              if area.delete.isNil: area.delete = expandDirs(e.value)
            else:
              quit("Unknown area configuration: " & e.key)
74272cdb   Göran Krampe   First commit
173
174
175
176
177
178
        of cfgOption:
          quit("Unknown configuration: " & e.key)
        of cfgError:
          quit("Parsing " & filename & ": " & e.msg)
      close(p)
  
db95b901   Göran Krampe   Various fixes, mo...
179
  # Trivial helper to enable verbose
ba6047f5   Göran Krampe   Added areas, move...
180
  proc run(cmd: string): int =
db95b901   Göran Krampe   Various fixes, mo...
181
182
    if verbose: echo(cmd)
    execCmd(cmd)
74272cdb   Göran Krampe   First commit
183
  
6884d534   Göran Krampe   Fixed unixy path ...
184
185
186
187
188
189
190
191
192
193
194
  # Perhaps not perfect but tries to convert c:\foo\bar to /c/foo/bar
  # but only if we are on windows, otherwise we do nothing.
  proc toUnixyPath(path: string): string =
    when defined(windows):
      if path.len > 1 and path[1] == ':':
        let parts = path.split({':', '\\'})
        result = "/" & parts.join("/")
      else:
        result = path
    else:
      result = path
ba6047f5   Göran Krampe   Added areas, move...
195
196
197
198
199
  
  # Upload a file to the remote area
  proc uploadFile(filename, fromDir: string) =
    if remoteArea.isNil:
      echo("Remote area not set in configuration file, skipping uploading content:\n\t" & filename)
74272cdb   Göran Krampe   First commit
200
      return
6884d534   Göran Krampe   Fixed unixy path ...
201
    let errorCode = run(format(remoteArea.upload, filename, remoteArea.url, toUnixyPath(fromDir)))
b9ad52ff   Göran Krampe   Added default con...
202
    if errorCode != 0:
ba6047f5   Göran Krampe   Added areas, move...
203
      quit("Something went wrong uploading " & filename & " to " & remoteArea.url, 2)
74272cdb   Göran Krampe   First commit
204
    
ba6047f5   Göran Krampe   Added areas, move...
205
206
207
208
  # Download a file to the remote area
  proc downloadFile(filename, toDir: string) =
    if remoteArea.isNil:
      quit("Remote area not set in configuration file, can not download content:\n\t" & filename)
6884d534   Göran Krampe   Fixed unixy path ...
209
    let errorCode = run(format(remoteArea.download, filename, remoteArea.url, toUnixyPath(toDir)))
b9ad52ff   Göran Krampe   Added default con...
210
    if errorCode != 0:
ba6047f5   Göran Krampe   Added areas, move...
211
      quit("Something went wrong downloading " & filename & " from " & remoteArea.url, 3)
5fc367ad   Göran Krampe   Renamed to blimp,...
212
  
e914743a   Göran Krampe   Added remove comm...
213
  # Delete a file from the remote master blimpStore
ba6047f5   Göran Krampe   Added areas, move...
214
215
  proc remoteDeleteFile(filename: string) =
    if remoteArea.isNil:
e914743a   Göran Krampe   Added remove comm...
216
      return
6884d534   Göran Krampe   Fixed unixy path ...
217
    let errorCode = run(format(remoteArea.delete, filename, remoteArea.url, toUnixyPath(blimpStore)))
e914743a   Göran Krampe   Added remove comm...
218
    if errorCode != 0:
ba6047f5   Göran Krampe   Added areas, move...
219
      quit("Something went wrong deleting " & filename & " from " & remoteArea.url, 3)
5fc367ad   Göran Krampe   Renamed to blimp,...
220
  
3e2f4c14   Göran Krampe   Added upload comm...
221
  # Copy content to blimpStore and upload if it was a new file or upload == true.
5fc367ad   Göran Krampe   Renamed to blimp,...
222
223
  proc copyToBlimpStore(filename, blimpFilename: string) =
    if not existsFile(blimpStore / blimpFilename):
aaf9bcef   Göran Krampe   Upped to version ...
224
225
226
227
228
229
230
      if stdio:
        try:
          writeFile(blimpStore / blimpFilename, stdinContent)
        except:
          quit("Failed writing file: " & blimpStore / blimpFilename & " from stdin", 1)
      else:
        copyFile(filename, blimpStore / blimpFilename)
ba6047f5   Göran Krampe   Added areas, move...
231
      uploadFile(blimpFilename, blimpStore)
3e2f4c14   Göran Krampe   Added upload comm...
232
    
5fc367ad   Göran Krampe   Renamed to blimp,...
233
234
  
  # Copy content from blimpStore, and downloading first if needed
e914743a   Göran Krampe   Added remove comm...
235
  proc copyFromBlimpStore(blimpFilename, filename: string) =
5fc367ad   Göran Krampe   Renamed to blimp,...
236
    if not existsFile(blimpStore / blimpFilename):
ba6047f5   Göran Krampe   Added areas, move...
237
      downloadFile(blimpFilename, blimpStore)
aaf9bcef   Göran Krampe   Upped to version ...
238
239
240
241
242
243
244
245
    if stdio:
      try:
        var content = readFile(blimpStore / blimpFilename)
        write(stdout, content)
      except:
        quit("Failed reading file: " & blimpStore / blimpFilename & " to stdout", 1)
    else:
      copyFile(blimpStore / blimpFilename, filename)
74272cdb   Göran Krampe   First commit
246
  
e914743a   Göran Krampe   Added remove comm...
247
248
249
250
251
252
  # Delete from blimpStore and remote.
  proc deleteFromBlimpStore(blimpFilename, filename: string) =
    if existsFile(blimpStore / blimpFilename):
      removeFile(blimpStore / blimpFilename)
    remoteDeleteFile(blimpFilename)
  
aaf9bcef   Göran Krampe   Upped to version ...
253
  proc blimpFileNameFromString(line: string): string =
386557e6   Göran Krampe   Added support for...
254
255
    let hashline = split(strip(line), ':')
    if hashline[0] == "blimphash":
e914743a   Göran Krampe   Added remove comm...
256
257
258
259
      result = hashline[1]
    else:
      result = nil
  
3e2f4c14   Göran Krampe   Added upload comm...
260
  # Pick out blimpFilename (filename & "-" & hash) or nil
aaf9bcef   Göran Krampe   Upped to version ...
261
262
263
264
265
266
267
268
269
  proc blimpFileName(filename: string): string =
    if stdio:
      blimpFileNameFromString(stdinContent)
    else:
      var hashfile: File
      if not open(hashfile, filename):
        quit("Failed opening file: " & filename, 4)
      blimpFileNameFromString(string(readLine(hashfile)))  
  
e914743a   Göran Krampe   Added remove comm...
270
271
  # Get hash and compute blimpFilename
  proc computeBlimpFilename(filename: string): string =
dab0c5b0   Göran Krampe   Added --version, ...
272
273
274
275
276
277
    var content: string
    try:
      content = readFile(filename)
    except:
      quit("Failed opening file: " & filename, 1)
    let hash = getMD5(content)
14d16aa7   Göran Krampe   Now the git filte...
278
    result = extractFilename(filename) & "-" & hash
e914743a   Göran Krampe   Added remove comm...
279
   
5fc367ad   Göran Krampe   Renamed to blimp,...
280
  # Copy original file to blimpStore and replace with hash stub in git.
74272cdb   Göran Krampe   First commit
281
  proc deflate(filename: string) =
dab0c5b0   Göran Krampe   Added --version, ...
282
283
284
285
286
287
288
    if verbose: echo "Deflating " & filename
    var blimpFilename = blimpFilename(filename)
    if not blimpFilename.isNil:
      echo("\t" & filename & " is already deflated, skipping.")
    else:
      blimpFilename = computeBlimpFilename(filename)
      copyToBlimpStore(filename, blimpFilename)
aaf9bcef   Göran Krampe   Upped to version ...
289
      if stdio:
386557e6   Göran Krampe   Added support for...
290
        write(stdout, "blimphash:" & blimpFilename)
aaf9bcef   Göran Krampe   Upped to version ...
291
      else:
386557e6   Göran Krampe   Added support for...
292
        writeFile(filename, "blimphash:" & blimpFilename)
dab0c5b0   Göran Krampe   Added --version, ...
293
      if verbose: echo("\t" & filename & " deflated.")
e914743a   Göran Krampe   Added remove comm...
294
  
386557e6   Göran Krampe   Added support for...
295
296
  # Iterator over all deflated files in the git clone
  iterator allDeflated() =
7e507b62   Göran Krampe   Fixes discovered ...
297
    let filenames = cmd("git ls-files " & gitRootDir).split({'\l', '\c'})
386557e6   Göran Krampe   Added support for...
298
299
300
    for fn in filenames:
      if not blimpFilename(fn).isNil:
        yield fn
ec7d9613   Göran Krampe   Added option --fi...
301
302
303
        
  # Iterator over all files matching the blimp filter in the git clone
  iterator allFiltered() =
7e507b62   Göran Krampe   Fixes discovered ...
304
    let lines = cmd("git ls-files | git check-attr --stdin filter").split({'\l', '\c'})
ec7d9613   Göran Krampe   Added option --fi...
305
306
307
308
    for line in lines:
      let status = line.split(':')
      if strip(status[2]) == "blimp":
        yield status[0]
386557e6   Göran Krampe   Added support for...
309
  
5fc367ad   Göran Krampe   Renamed to blimp,...
310
  # Parse out hash from hash stub and copy back original content from blimpStore.
74272cdb   Göran Krampe   First commit
311
  proc inflate(filename: string) =
dab0c5b0   Göran Krampe   Added --version, ...
312
    if verbose: echo "Inflating " & filename
e914743a   Göran Krampe   Added remove comm...
313
314
    let blimpFilename = blimpFilename(filename)
    if blimpFilename.isNil:
dab0c5b0   Göran Krampe   Added --version, ...
315
      echo("\t" & filename & " is not deflated, skipping.")
e914743a   Göran Krampe   Added remove comm...
316
317
    else:
      copyFromBlimpStore(blimpfilename, filename)
dab0c5b0   Göran Krampe   Added --version, ...
318
      if verbose: echo("\t" & filename & " inflated.")
e914743a   Göran Krampe   Added remove comm...
319
320
321
322
323
324
325
  
  # Inflates file first (if deflated) and then removes current content for it,
  # both locally and in remote.
  proc remove(filename: string) =
    var blimpFilename = blimpFilename(filename)
    if not blimpFilename.isNil:
      copyFromBlimpStore(blimpfilename, filename)
74272cdb   Göran Krampe   First commit
326
    else:
e914743a   Göran Krampe   Added remove comm...
327
328
329
      blimpFilename = computeBlimpFilename(filename)
    deleteFromBlimpStore(blimpfilename, filename)
    echo("\t" & filename & " content removed from blimpstore locally and remotely.")
5fc367ad   Göran Krampe   Renamed to blimp,...
330
  
dab0c5b0   Göran Krampe   Added --version, ...
331
  
ba6047f5   Göran Krampe   Added areas, move...
332
333
  # Make sure a file already in blimpstore is uploaded to remote area.
  proc push(filename: string) =
3e2f4c14   Göran Krampe   Added upload comm...
334
335
336
337
338
    if verbose: echo "Uploading " & filename
    var blimpFilename = blimpFilename(filename)
    if blimpFilename.isNil:
      blimpFilename = computeBlimpFilename(filename)
    if existsFile(blimpStore / blimpFilename):
ba6047f5   Göran Krampe   Added areas, move...
339
      uploadFile(blimpFilename, blimpStore)
3e2f4c14   Göran Krampe   Added upload comm...
340
341
342
343
      if verbose: echo("\t" & filename & " uploaded.")
    else:
      if verbose: echo("\t" & filename & " is not in blimpstore, skipping.")
  
ba6047f5   Göran Krampe   Added areas, move...
344
345
346
347
348
349
350
351
352
353
354
  # Upload a file to a remote area
  proc upload(filename: string) =
    if verbose: echo "Uploading " & filename
    uploadFile(filename, currentDir)
    if verbose: echo("\t" & filename & " uploaded.")
      
  # Download a file from a remote area.
  proc download(filename: string) =
    if verbose: echo "Downloading " & filename
    downloadFile(filename, currentDir)
    if verbose: echo("\t" & filename & " downloaded.")
3e2f4c14   Göran Krampe   Added upload comm...
355
  
dab0c5b0   Göran Krampe   Added --version, ...
356
357
358
359
360
361
362
363
364
365
366
367
368
  proc setupBlimpStore() =
    try:
      if not existsDir(blimpStore):
        createDir(blimpStore)
    except:
      quit("Could not create " & blimpStore & " directory.", 1)
  
    try:
      if not existsFile(blimpStore / ".blimp.conf"):
        writeFile(blimpStore / ".blimp.conf", defaultConfig)
    except:
      quit("Could not create .blimp.conf config file in " & blimpStore & " directory.", 1)
  
78569137   Göran Krampe   Added directory e...
369
370
371
  proc `$`(x: string): string =
    if x.isNil: "nil" else: x
  
dab0c5b0   Göran Krampe   Added --version, ...
372
373
374
  proc dumpConfig() =
    echo "\nDump of configuration:"
    echo "\tblimpStore: " & blimpStore
ba6047f5   Göran Krampe   Added areas, move...
375
376
377
    echo "\tremote-url: " & remoteArea.url
    echo "\tremote-upload: " & remoteArea.upload
    echo "\tremote-download: " & remoteArea.download
aaf9bcef   Göran Krampe   Upped to version ...
378
    echo "\tblimpVersion: " & $blimpVersion
dab0c5b0   Göran Krampe   Added --version, ...
379
    echo "\n"
74272cdb   Göran Krampe   First commit
380
  
386557e6   Göran Krampe   Added support for...
381
  let synopsis = """
db95b901   Göran Krampe   Various fixes, mo...
382
    blimp [options] <command> <filenames...>
ba6047f5   Göran Krampe   Added areas, move...
383
384
385
      -h,--help                  Show this
      --version                  Show version of blimp
      -v,--verbose               Verbosity, only works without -s
386557e6   Göran Krampe   Added support for...
386
      ----------
ba6047f5   Göran Krampe   Added areas, move...
387
388
389
390
      <command>  (string)        (d)eflate, (i)nflate, init, remove, push, upload, download
      --all                      Operate on all deflated files in clone
      -f,--filter                Operate on all files matching blimp filter
      -a,--area (default remote) The area to use for remote up/downloads
386557e6   Göran Krampe   Added support for...
391
      ----------
ba6047f5   Göran Krampe   Added areas, move...
392
393
      -s,--stdio                 If given, use stdin/stdout for content.
      <filenames> (string...)    One or more filepaths to inflate/deflate
386557e6   Göran Krampe   Added support for...
394
395
  """
  let help = """
3e2f4c14   Göran Krampe   Added upload comm...
396
  
aaf9bcef   Göran Krampe   Upped to version ...
397
398
399
    blimp is a little utility program for handling large files
    in git repositories. Its inspired by git-fat and s3annex
    but doesn't rely on S3 for storage - it uses rsync like git-fat.
386557e6   Göran Krampe   Added support for...
400
401
    It is a single binary without any dependencies. Its not as advanced
    as git-fat but basically does the same thing.
aaf9bcef   Göran Krampe   Upped to version ...
402
403
404
  
    Manual use:
  
386557e6   Göran Krampe   Added support for...
405
406
    Use "blimp d mybigfile" to deflate a file, typically before commit.
    Use "blimp i mybigfile" to inflate it back to original content.
aaf9bcef   Göran Krampe   Upped to version ...
407
  
3e2f4c14   Göran Krampe   Added upload comm...
408
409
    When deflated the file only has this content:
    
386557e6   Göran Krampe   Added support for...
410
      "blimphash:" <filename> "-" <md5sum>
aaf9bcef   Göran Krampe   Upped to version ...
411
  
386557e6   Göran Krampe   Added support for...
412
    Deflate also copies the real content to your local blimpstore:
aaf9bcef   Göran Krampe   Upped to version ...
413
    
386557e6   Göran Krampe   Added support for...
414
      <blimpstore>/<filename>-<md5sum>
aaf9bcef   Göran Krampe   Upped to version ...
415
  
386557e6   Göran Krampe   Added support for...
416
    ...and if configured also uploads it to "remote", using rsync.
aaf9bcef   Göran Krampe   Upped to version ...
417
418
419
420
  
    Configuration is in these locations in order:
   
      ./.blimp.conf
386557e6   Göran Krampe   Added support for...
421
      <gitroot>/.blimp.conf
aaf9bcef   Göran Krampe   Upped to version ...
422
423
424
425
426
427
428
429
430
431
432
433
      ~/<blimpstore>/.blimp.conf
      ~/.blimp.conf
  
    This way you can have settings per directory, per git clone,
    per store and per user. A default blimpstore with a commented .blimp.conf
    is created in ~/blimpstore if you run blimp and no .blimp.conf is found.
  
    Edit ~/blimpstore/.blimp.conf (or in another location) and set a proper
    remote and the proper rsync password. This ensures that its also properly
    synced with a master rsync repository that is typically shared.
  
    Inflate will bring back the original content by copying from
386557e6   Göran Krampe   Added support for...
434
    your local blimpstore, and if its not there, first download from the remote.
aaf9bcef   Göran Krampe   Upped to version ...
435
436
437
438
439
    Use this whenever you need to work/edit the big file - in order to get
    its real content.
  
    The filenames given are all processed. If -s is used content is processed via
    stdin/stdout and only one filename can be passed. This is used when running blimp
386557e6   Göran Krampe   Added support for...
440
    via a git filter (smudge/clean), see below.
aaf9bcef   Göran Krampe   Upped to version ...
441
442
443
444
445
  
    The remove command (no single character shortcut) will remove the file(s) content
    both from the local blimpstore and from the remote. This only removes
    the current content version, not older versions. The file itself is first
    inflated, if needed, and not deleted. This only "unblimps" the file.
386557e6   Göran Krampe   Added support for...
446
    
ba6047f5   Göran Krampe   Added areas, move...
447
    The push command (no single character shortcut) will force upload the given file
3e2f4c14   Göran Krampe   Added upload comm...
448
    from the local blimpstore to the remote. This is normally done automatically,
ba6047f5   Göran Krampe   Added areas, move...
449
450
451
452
    but this way you can make sure they are synced onto the remote.
    
    The upload and download commands are used to distribute artifacts typically in a
    build script. If no --area is given, they use the standard "remote" area.
3e2f4c14   Göran Krampe   Added upload comm...
453
    
386557e6   Göran Krampe   Added support for...
454
455
456
    In order to have blimp work automatically you can:
    
    * Create a .gitattributes file with lines like:
ec7d9613   Göran Krampe   Added option --fi...
457
      *.png filter=blimp binary
386557e6   Göran Krampe   Added support for...
458
459
460
461
462
      
    * Configure blimp as a filter by running:
      git config filter.blimp.clean "blimp -s d %f"
      git config filter.blimp.smudge "blimp -s i %f"
  
ba6047f5   Göran Krampe   Added areas, move...
463
    The above git config can be done by running "blimp init" just like git.
386557e6   Göran Krampe   Added support for...
464
465
466
467
468
469
470
  
    When the above is done (per clone) git will automatically run blimp deflate
    just before committing and blimp inflate when operations are done.
    
    This means that if you clone a git repository that already has a .gitattributes
    file in it that uses the blimp filter, then you should do:
    
ba6047f5   Göran Krampe   Added areas, move...
471
472
      blimp init
      blimp inflate --filter
386557e6   Göran Krampe   Added support for...
473
    
ba6047f5   Göran Krampe   Added areas, move...
474
    This will configure the blimp filter and then find and inflate all deflated
386557e6   Göran Krampe   Added support for...
475
    files throughout the clone.
aaf9bcef   Göran Krampe   Upped to version ...
476
  """
db95b901   Göran Krampe   Various fixes, mo...
477
  
74272cdb   Göran Krampe   First commit
478
  ################################ main #####################################
78569137   Göran Krampe   Added directory e...
479
480
481
482
483
  # Set some dirs
  homeDir = getHomeDir()
  homeDir = homeDir[0.. -2] # Not sure why it keeps a trailing "/" on Linux
  currentDir = getCurrentDir()
  gitRootDir = gitRoot()
74272cdb   Göran Krampe   First commit
484
  
dab0c5b0   Göran Krampe   Added --version, ...
485
  # Using lapp to get args, on parsing failure this will show usage automatically
386557e6   Göran Krampe   Added support for...
486
  var args = parse(synopsis)
dab0c5b0   Göran Krampe   Added --version, ...
487
  verbose = args["verbose"].asBool
aaf9bcef   Göran Krampe   Upped to version ...
488
  stdio = args["stdio"].asBool
ec7d9613   Göran Krampe   Added option --fi...
489
490
  onAllDeflated = args["all"].asBool
  onAllFiltered = args["filter"].asBool
ba6047f5   Göran Krampe   Added areas, move...
491
  area = args["area"].asString
aaf9bcef   Göran Krampe   Upped to version ...
492
493
494
495
496
497
498
499
500
501
502
  
  # Can't do verbose with -s, that messes up stdout,
  # read in all of stdin once and for all
  if stdio:
    verbose = false
    try:
      stdinContent = readAll(stdin)
      close(stdin)
    except:
      quit("Failed reading stdin", 1)
  
dab0c5b0   Göran Krampe   Added --version, ...
503
  # Parse configuration files, may shadow and override each other
78569137   Göran Krampe   Added directory e...
504
  parseConfFile(currentDir / ".blimp.conf")
386557e6   Göran Krampe   Added support for...
505
  if not gitRootDir.isNil and gitRootDir != currentDir:
78569137   Göran Krampe   Added directory e...
506
    parseConfFile(gitRootDir / ".blimp.conf")
74272cdb   Göran Krampe   First commit
507
  
ba6047f5   Göran Krampe   Added areas, move...
508
509
510
511
512
  if existsDir(homeDir / "blimpstore"):
    parseConfFile(homeDir / "blimpstore" / ".blimp.conf")
  
  parseConfFile(homeDir / ".blimp.conf")
  
dab0c5b0   Göran Krampe   Added --version, ...
513
514
  # If we haven't gotten a blimpstore yet, we set a default one
  if blimpStore.isNil:
78569137   Göran Krampe   Added directory e...
515
    blimpStore = homeDir / "blimpstore"
74272cdb   Göran Krampe   First commit
516
  
ba6047f5   Göran Krampe   Added areas, move...
517
518
519
  # Check if we have a configured remoteArea
  if areas.hasKey(area):
    remoteArea = areas[area]
74272cdb   Göran Krampe   First commit
520
  
386557e6   Göran Krampe   Added support for...
521
  # Let's just show what we have :)
dab0c5b0   Göran Krampe   Added --version, ...
522
  if verbose: dumpConfig()
e914743a   Göran Krampe   Added remove comm...
523
  
dab0c5b0   Göran Krampe   Added --version, ...
524
  # These two are special, they short out
386557e6   Göran Krampe   Added support for...
525
  if args.showHelp: quit(synopsis & help)
dab0c5b0   Göran Krampe   Added --version, ...
526
  if args.showVersion: quit("blimp version: " & versionAsString)
74272cdb   Göran Krampe   First commit
527
  
aaf9bcef   Göran Krampe   Upped to version ...
528
529
530
531
  # Check blimpVersion
  if not blimpVersion.isNil and blimpVersion != versionAsString:
    quit("Wrong version of blimp, configuration wants: " & blimpVersion)
  
386557e6   Göran Krampe   Added support for...
532
  
ba6047f5   Göran Krampe   Added areas, move...
533
  # Ok, let's see 
3e2f4c14   Göran Krampe   Added upload comm...
534
535
536
  var filenames = initSet[string]()
  
  # Add upp all files to operate on in a Set
386557e6   Göran Krampe   Added support for...
537
  if args.hasKey("filenames"):
3e2f4c14   Göran Krampe   Added upload comm...
538
539
540
541
542
543
544
545
546
    for f in args["filenames"].asSeq:
      if f.asString != "":
        filenames.incl(f.asString)
  if onAllDeflated:
    for fn in allDeflated():
      filenames.incl(fn)
  if onAllFiltered:
    for fn in allFiltered():
      filenames.incl(fn)
dab0c5b0   Göran Krampe   Added --version, ...
547
548
549
550
  
  # Make sure the local blimpstore is setup.
  setupBlimpStore()
  
ba6047f5   Göran Krampe   Added areas, move...
551
  
74272cdb   Göran Krampe   First commit
552
  # Do the deed
ba6047f5   Göran Krampe   Added areas, move...
553
  let command = args["command"].asString
386557e6   Göran Krampe   Added support for...
554
  if command != "":
ba6047f5   Göran Krampe   Added areas, move...
555
556
557
558
559
560
    # Should we install the filter?
    if command == "init":
      ensureBlimpFilter()
      echo("Installed blimp filter")
      quit(0)
    elif command == "d" or command == "deflate":
3e2f4c14   Göran Krampe   Added upload comm...
561
562
      for fn in filenames:
        deflate(fn)
386557e6   Göran Krampe   Added support for...
563
    elif command == "i" or command == "inflate":
3e2f4c14   Göran Krampe   Added upload comm...
564
565
      for fn in filenames:
        inflate(fn)
386557e6   Göran Krampe   Added support for...
566
567
    elif command == "remove":
      for fn in filenames:
3e2f4c14   Göran Krampe   Added upload comm...
568
        remove(fn)
ba6047f5   Göran Krampe   Added areas, move...
569
570
571
    elif command == "push":
      for fn in filenames.items:
        push(fn)
3e2f4c14   Göran Krampe   Added upload comm...
572
    elif command == "upload":
3e2f4c14   Göran Krampe   Added upload comm...
573
574
      for fn in filenames.items:
        upload(fn)
ba6047f5   Göran Krampe   Added areas, move...
575
576
577
    elif command == "download":
      for fn in filenames.items:
        download(fn)
386557e6   Göran Krampe   Added support for...
578
    else:
ba6047f5   Göran Krampe   Added areas, move...
579
      quit("Unknown command: \"" & command & "\", use --help for valid commands.", 6)
74272cdb   Göran Krampe   First commit
580
581
582
  
  # All good
  quit(0)