Blame view

blimp.nim 10.4 KB
b9ad52ff   Göran Krampe   Added default con...
1
  import md5, os, osproc, parseopt2, strutils, parsecfg, streams, lapp, subexes
74272cdb   Göran Krampe   First commit
2
  
5fc367ad   Göran Krampe   Renamed to blimp,...
3
  # blimp is a little utility program for handling large files
74272cdb   Göran Krampe   First commit
4
  # in git repositories. Its inspired by git-fat and s3annex
5fc367ad   Göran Krampe   Renamed to blimp,...
5
6
  # but doesn't rely on S3 for storage, is a single binary without
  # need for Python, and has less features than git-fat. So far.
74272cdb   Göran Krampe   First commit
7
  #
5fc367ad   Göran Krampe   Renamed to blimp,...
8
9
10
11
  # Manual use:
  #
  # Use "blimp d mybigfile" to deflate it before commit.
  # Use "blimp i mybigfile" to inflate it back to original size.
74272cdb   Göran Krampe   First commit
12
13
14
15
  #
  # When deflated the file only has an md5sum string inside it.
  #
  # The file is copied over into:
5fc367ad   Göran Krampe   Renamed to blimp,...
16
17
  #  <homedir>/blimpStore/<originalfilename>-<md5sum>
  #
dab0c5b0   Göran Krampe   Added --version, ...
18
19
  # Configuration is in these locations in order:
  #   ./.blimp.conf
5fc367ad   Göran Krampe   Renamed to blimp,...
20
21
  #   <gitroot>/.blimp.conf
  #   ~/blimpstore/.blimp.conf
dab0c5b0   Göran Krampe   Added --version, ...
22
23
24
25
26
27
28
  #   ~/.blimp.conf
  
  const 
    versionMajor* = 0
    versionMinor* = 2
    versionPatch* = 1
    versionAsString* = $versionMajor & "." & $versionMinor & "." & $versionPatch
74272cdb   Göran Krampe   First commit
29
  
b9ad52ff   Göran Krampe   Added default con...
30
  var
dab0c5b0   Göran Krampe   Added --version, ...
31
    blimpStore, remoteBlimpStore, uploadCommandFormat, downloadCommandFormat, deleteCommandFormat, rsyncPassword: string = nil
b9ad52ff   Göran Krampe   Added default con...
32
    verbose: bool
74272cdb   Göran Krampe   First commit
33
  
b9ad52ff   Göran Krampe   Added default con...
34
35
36
  let
    defaultConfig = """
  [rsync]
e914743a   Göran Krampe   Added remove comm...
37
  # Set this to your remote rsync daemon area
dab0c5b0   Göran Krampe   Added --version, ...
38
  remote = "blimpuser@some-rsync-server.com::blimpstore"
dab0c5b0   Göran Krampe   Added --version, ...
39
  password = "some-good-rsync-password-for-blimpuser"
e914743a   Göran Krampe   Added remove comm...
40
41
  
  # The following three formats should not need editing
b9ad52ff   Göran Krampe   Added default con...
42
  # $1 is filename, $2 is remote and $3 is the local blimpstore
dab0c5b0   Göran Krampe   Added --version, ...
43
44
  upload = "rsync --password-file $3/.blimp.pass -avzP $3/$1 $2/"
  download = "rsync --password-file $3/.blimp.pass -avzP $2/$1 $3/"
e914743a   Göran Krampe   Added remove comm...
45
  # This deletes a single file from destination, that is already deleted in source
dab0c5b0   Göran Krampe   Added --version, ...
46
  delete = "rsync --password-file $3/.blimp.pass -dv --delete --existing --ignore-existing --include '$1' --exclude '*' $3/ $2"
b9ad52ff   Göran Krampe   Added default con...
47
  """
74272cdb   Göran Krampe   First commit
48
  
dab0c5b0   Göran Krampe   Added --version, ...
49
  # Load a blimp.conf file
74272cdb   Göran Krampe   First commit
50
  proc parseConfFile(filename: string) =
74272cdb   Göran Krampe   First commit
51
52
    var f = newFileStream(filename, fmRead)
    if f != nil:
dab0c5b0   Göran Krampe   Added --version, ...
53
      if verbose: echo "Reading config: " & filename
74272cdb   Göran Krampe   First commit
54
      var p: CfgParser
dab0c5b0   Göran Krampe   Added --version, ...
55
      
74272cdb   Göran Krampe   First commit
56
57
58
59
60
61
62
63
64
      open(p, f, filename)
      while true:
        var e = next(p)
        case e.kind
        of cfgEof: 
          break
        of cfgSectionStart:
          continue # Ignore
        of cfgKeyValuePair:
b9ad52ff   Göran Krampe   Added default con...
65
          case e.key
dab0c5b0   Göran Krampe   Added --version, ...
66
67
          of "blimpstore":
            if blimpStore.isNil: blimpStore = e.value
b9ad52ff   Göran Krampe   Added default con...
68
          of "remote":
dab0c5b0   Göran Krampe   Added --version, ...
69
70
71
            if remoteBlimpStore.isNil: remoteBlimpStore = e.value
          of "password":
            if rsyncPassword.isNil: rsyncPassword = e.value
b9ad52ff   Göran Krampe   Added default con...
72
          of "upload":
dab0c5b0   Göran Krampe   Added --version, ...
73
            if uploadCommandFormat.isNil: uploadCommandFormat = e.value
b9ad52ff   Göran Krampe   Added default con...
74
          of "download":
dab0c5b0   Göran Krampe   Added --version, ...
75
            if downloadCommandFormat.isNil: downloadCommandFormat = e.value
e914743a   Göran Krampe   Added remove comm...
76
          of "delete":
dab0c5b0   Göran Krampe   Added --version, ...
77
            if deleteCommandFormat.isNil: deleteCommandFormat = e.value
74272cdb   Göran Krampe   First commit
78
79
80
81
82
83
84
85
          else:
            quit("Unknown configuration: " & e.key)
        of cfgOption:
          quit("Unknown configuration: " & e.key)
        of cfgError:
          quit("Parsing " & filename & ": " & e.msg)
      close(p)
  
db95b901   Göran Krampe   Various fixes, mo...
86
87
88
89
  # Trivial helper to enable verbose
  proc run(cmd: string): auto =
    if verbose: echo(cmd)
    execCmd(cmd)
74272cdb   Göran Krampe   First commit
90
  
dab0c5b0   Göran Krampe   Added --version, ...
91
92
93
94
95
96
97
  # Every rsync command, make sure we have a password file
  proc rsyncRun(cmd: string): auto =
    writeFile(blimpStore / ".blimp.pass", rsyncPassword)
    if execCmd("chmod 600 " & blimpStore / ".blimp.pass") != 0:
      quit("Failed to chmod 600 " & blimpStore / ".blimp.pass")
    run(cmd)
  
5fc367ad   Göran Krampe   Renamed to blimp,...
98
99
100
  # Upload a file to the remote master blimpStore
  proc uploadFile(blimpFilename: string) =
    if remoteBlimpStore.isNil:
e914743a   Göran Krampe   Added remove comm...
101
      echo("Remote blimpstore not set in configuration file, skipping uploading content:\n\t" & blimpFilename)
74272cdb   Göran Krampe   First commit
102
      return
dab0c5b0   Göran Krampe   Added --version, ...
103
    let errorCode = rsyncRun(format(uploadCommandFormat, blimpFilename, remoteBlimpStore, blimpStore))
b9ad52ff   Göran Krampe   Added default con...
104
105
    if errorCode != 0:
      quit("Something went wrong uploading " & blimpFilename & " to " & remoteBlimpStore, 2)
74272cdb   Göran Krampe   First commit
106
    
5fc367ad   Göran Krampe   Renamed to blimp,...
107
108
109
110
  # Download a file to the remote master blimpStore
  proc downloadFile(blimpFilename: string) =
    if remoteBlimpStore.isNil:
      quit("Remote blimpstore not set in configuration file, can not download content:\n\t" & blimpFilename)
dab0c5b0   Göran Krampe   Added --version, ...
111
    let errorCode = rsyncRun(format(downloadCommandFormat, blimpFilename, remoteBlimpStore, blimpStore))
b9ad52ff   Göran Krampe   Added default con...
112
113
    if errorCode != 0:
      quit("Something went wrong downloading " & blimpFilename & " from " & remoteBlimpStore, 3)
5fc367ad   Göran Krampe   Renamed to blimp,...
114
  
e914743a   Göran Krampe   Added remove comm...
115
116
117
118
  # Delete a file from the remote master blimpStore
  proc remoteDeleteFile(blimpFilename: string) =
    if remoteBlimpStore.isNil:
      return
dab0c5b0   Göran Krampe   Added --version, ...
119
    let errorCode = rsyncRun(format(deleteCommandFormat, blimpFilename, remoteBlimpStore, blimpStore))
e914743a   Göran Krampe   Added remove comm...
120
121
    if errorCode != 0:
      quit("Something went wrong deleting " & blimpFilename & " from " & remoteBlimpStore, 3)
5fc367ad   Göran Krampe   Renamed to blimp,...
122
123
124
125
126
127
128
129
  
  # Copy content to blimpStore, no upload yet.
  proc copyToBlimpStore(filename, blimpFilename: string) =
    if not existsFile(blimpStore / blimpFilename):
      copyFile(filename, blimpStore / blimpFilename)
      uploadFile(blimpFilename)
  
  # Copy content from blimpStore, and downloading first if needed
e914743a   Göran Krampe   Added remove comm...
130
  proc copyFromBlimpStore(blimpFilename, filename: string) =
5fc367ad   Göran Krampe   Renamed to blimp,...
131
132
133
    if not existsFile(blimpStore / blimpFilename):
      downloadFile(blimpFilename)
    copyFile(blimpStore / blimpFilename, filename)
74272cdb   Göran Krampe   First commit
134
  
e914743a   Göran Krampe   Added remove comm...
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
  # Delete from blimpStore and remote.
  proc deleteFromBlimpStore(blimpFilename, filename: string) =
    if existsFile(blimpStore / blimpFilename):
      removeFile(blimpStore / blimpFilename)
    remoteDeleteFile(blimpFilename)
  
  # Pick out blimpFilename (filename & "-" & hash)
  proc blimpFileName(filename: string): string=
    var hashfile: File
    if not open(hashfile, filename):
      quit("Failed opening file: " & filename, 4)
    let hashline = split(string(readLine(hashfile)), {':'})
    if hashline[0] == "hash":
      result = hashline[1]
    else:
      result = nil
  
  # Get hash and compute blimpFilename
  proc computeBlimpFilename(filename: string): string =
dab0c5b0   Göran Krampe   Added --version, ...
154
155
156
157
158
159
160
    var content: string
    try:
      content = readFile(filename)
    except:
      quit("Failed opening file: " & filename, 1)
    let hash = getMD5(content)
    result = filename & "-" & hash
e914743a   Göran Krampe   Added remove comm...
161
   
5fc367ad   Göran Krampe   Renamed to blimp,...
162
  # Copy original file to blimpStore and replace with hash stub in git.
74272cdb   Göran Krampe   First commit
163
  proc deflate(filename: string) =
dab0c5b0   Göran Krampe   Added --version, ...
164
165
166
167
168
169
170
171
172
    if verbose: echo "Deflating " & filename
    var blimpFilename = blimpFilename(filename)
    if not blimpFilename.isNil:
      echo("\t" & filename & " is already deflated, skipping.")
    else:
      blimpFilename = computeBlimpFilename(filename)
      copyToBlimpStore(filename, blimpFilename)
      writeFile(filename, "hash:" & blimpFilename)
      if verbose: echo("\t" & filename & " deflated.")
e914743a   Göran Krampe   Added remove comm...
173
  
5fc367ad   Göran Krampe   Renamed to blimp,...
174
  # Parse out hash from hash stub and copy back original content from blimpStore.
74272cdb   Göran Krampe   First commit
175
  proc inflate(filename: string) =
dab0c5b0   Göran Krampe   Added --version, ...
176
    if verbose: echo "Inflating " & filename
e914743a   Göran Krampe   Added remove comm...
177
178
    let blimpFilename = blimpFilename(filename)
    if blimpFilename.isNil:
dab0c5b0   Göran Krampe   Added --version, ...
179
      echo("\t" & filename & " is not deflated, skipping.")
e914743a   Göran Krampe   Added remove comm...
180
181
    else:
      copyFromBlimpStore(blimpfilename, filename)
dab0c5b0   Göran Krampe   Added --version, ...
182
      if verbose: echo("\t" & filename & " inflated.")
e914743a   Göran Krampe   Added remove comm...
183
184
185
186
187
188
189
  
  # Inflates file first (if deflated) and then removes current content for it,
  # both locally and in remote.
  proc remove(filename: string) =
    var blimpFilename = blimpFilename(filename)
    if not blimpFilename.isNil:
      copyFromBlimpStore(blimpfilename, filename)
74272cdb   Göran Krampe   First commit
190
    else:
e914743a   Göran Krampe   Added remove comm...
191
192
193
      blimpFilename = computeBlimpFilename(filename)
    deleteFromBlimpStore(blimpfilename, filename)
    echo("\t" & filename & " content removed from blimpstore locally and remotely.")
5fc367ad   Göran Krampe   Renamed to blimp,...
194
  
dab0c5b0   Göran Krampe   Added --version, ...
195
  # Find git root dir or nil
5fc367ad   Göran Krampe   Renamed to blimp,...
196
197
198
199
  proc gitRoot(): string =
    try:
      let tup = execCmdEx("git rev-parse --show-toplevel")
      if tup[1] == 0:
dab0c5b0   Göran Krampe   Added --version, ...
200
        result = strip(tup[0])
5fc367ad   Göran Krampe   Renamed to blimp,...
201
      else:
dab0c5b0   Göran Krampe   Added --version, ...
202
        result = nil
5fc367ad   Göran Krampe   Renamed to blimp,...
203
    except:
dab0c5b0   Göran Krampe   Added --version, ...
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
      result = nil
  
  proc setupBlimpStore() =
    try:
      if not existsDir(blimpStore):
        createDir(blimpStore)
    except:
      quit("Could not create " & blimpStore & " directory.", 1)
  
    try:
      if not existsFile(blimpStore / ".blimp.conf"):
        writeFile(blimpStore / ".blimp.conf", defaultConfig)
    except:
      quit("Could not create .blimp.conf config file in " & blimpStore & " directory.", 1)
  
  proc dumpConfig() =
    echo "\nDump of configuration:"
    echo "\tblimpStore: " & blimpStore
    echo "\tremoteBlimpStore: " & remoteBlimpStore
    echo "\tuploadCommandFormat: " & uploadCommandFormat
    echo "\tdownloadCommandFormat: " & downloadCommandFormat
    echo "\tdeleteCommandFormat: " & deleteCommandFormat
    echo "\trsyncPassword: " & rsyncPassword
    echo "\n"
74272cdb   Göran Krampe   First commit
228
  
db95b901   Göran Krampe   Various fixes, mo...
229
230
  let help = """
    blimp [options] <command> <filenames...>
dab0c5b0   Göran Krampe   Added --version, ...
231
232
      -h,--help                Show this
      --version                Show version of blimp
db95b901   Göran Krampe   Various fixes, mo...
233
      -v,--verbose             Verbosity
dab0c5b0   Göran Krampe   Added --version, ...
234
      <command>   (string)     (d)eflate, (i)nflate, remove
db95b901   Göran Krampe   Various fixes, mo...
235
      <filenames> (string...)  One or more filepaths to inflate/deflate
e914743a   Göran Krampe   Added remove comm...
236
237
      
      Edit ~/blimpstore/.blimp.conf or <gitroot>/.blimp.conf and set a proper
dab0c5b0   Göran Krampe   Added --version, ...
238
      remote and the proper rsync password to use.
e914743a   Göran Krampe   Added remove comm...
239
240
241
      
      Deflate is run before you add the big file to the index for committing.
      Deflate will replace the file contents with a hash, and copy the
dab0c5b0   Göran Krampe   Added --version, ...
242
      real content to your local blimpstore, and if configured also upload it to
e914743a   Göran Krampe   Added remove comm...
243
244
245
      remote, using rsync.
      
      Inflate will bring back the original content by copying from
dab0c5b0   Göran Krampe   Added --version, ...
246
      your local blimpstore, and if its not there, first downloading from the remote.
e914743a   Göran Krampe   Added remove comm...
247
248
249
250
      Use this whenever you need to work/edit the big file - in order to get
      its real content.
      
      Remove (no single character shortcut) will remove the file(s) content
dab0c5b0   Göran Krampe   Added --version, ...
251
      both from the local blimpstore and from the remote. This only removes
e914743a   Göran Krampe   Added remove comm...
252
253
      the current content version, not older versions. The file itself is first
      inflated, if needed, and not deleted. This only "unblimps" the file.
db95b901   Göran Krampe   Various fixes, mo...
254
255
    """
  
74272cdb   Göran Krampe   First commit
256
257
  ################################ main #####################################
  
dab0c5b0   Göran Krampe   Added --version, ...
258
259
260
261
262
263
264
265
  # Using lapp to get args, on parsing failure this will show usage automatically
  var args = parse(help)
  verbose = args["verbose"].asBool
  
  # Parse configuration files, may shadow and override each other
  parseConfFile(getCurrentDir() / ".blimp.conf")
  if not gitRoot().isNil:
    parseConfFile(gitRoot() / ".blimp.conf")
74272cdb   Göran Krampe   First commit
266
  
dab0c5b0   Göran Krampe   Added --version, ...
267
268
269
  # If we haven't gotten a blimpstore yet, we set a default one
  if blimpStore.isNil:
    blimpStore = getHomeDir() / "blimpstore"
74272cdb   Göran Krampe   First commit
270
  
dab0c5b0   Göran Krampe   Added --version, ...
271
272
273
  if existsDir(blimpStore):
    parseConfFile(blimpStore / ".blimp.conf")
  parseConfFile(getHomeDir() / ".blimp.conf")
74272cdb   Göran Krampe   First commit
274
  
dab0c5b0   Göran Krampe   Added --version, ...
275
  if verbose: dumpConfig()
e914743a   Göran Krampe   Added remove comm...
276
  
dab0c5b0   Göran Krampe   Added --version, ...
277
278
279
  # These two are special, they short out
  if args.showHelp: quit(help)
  if args.showVersion: quit("blimp version: " & versionAsString)
74272cdb   Göran Krampe   First commit
280
  
db95b901   Göran Krampe   Various fixes, mo...
281
282
  let command = args["command"].asString
  let filenames = args["filenames"].asSeq
dab0c5b0   Göran Krampe   Added --version, ...
283
284
285
286
287
  
  # Make sure the local blimpstore is setup.
  setupBlimpStore()
  
  
74272cdb   Göran Krampe   First commit
288
289
290
  
  # Do the deed
  if command == "d" or command == "deflate":
db95b901   Göran Krampe   Various fixes, mo...
291
292
    for fn in filenames:
      deflate(fn.asString)
74272cdb   Göran Krampe   First commit
293
  elif command == "i" or command == "inflate":
db95b901   Göran Krampe   Various fixes, mo...
294
295
    for fn in filenames:
      inflate(fn.asString)
e914743a   Göran Krampe   Added remove comm...
296
297
298
  elif command == "remove":
    for fn in filenames:
      remove(fn.asString)
74272cdb   Göran Krampe   First commit
299
300
301
302
303
  else:
    quit("Unknown command, only (d)eflate or (i)inflate are valid.", 6)
  
  # All good
  quit(0)