diff --git a/blimp.nim b/blimp.nim index 8ac59db..7388a89 100644 --- a/blimp.nim +++ b/blimp.nim @@ -20,17 +20,22 @@ import md5, os, osproc, parseopt2, strutils, parsecfg, streams, lapp, subexes # ~/blimpstore/.blimp.conf var - blimpStore, uploadCommandFormat, downloadCommandFormat: string + blimpStore, uploadCommandFormat, downloadCommandFormat, deleteCommandFormat: string remoteBlimpStore: string = nil verbose: bool let defaultConfig = """ [rsync] +# Set this to your remote rsync daemon area remote = "blimp@some-rsync-server.com::blimpstore" + +# The following three formats should not need editing # $1 is filename, $2 is remote and $3 is the local blimpstore upload = "rsync --password-file ~/blimp.pass -avzP $3/$1 $2/" -download = "rsync -avzP $2/$1 $3/" +download = "rsync --password-file ~/blimp.pass -avzP $2/$1 $3/" +# This deletes a single file from destination, that is already deleted in source +delete = "rsync --password-file ~/blimp.pass -dv --delete --existing --ignore-existing --include '$1' --exclude '*' $3/ $2" """ # Load blimp.conf file, overkill for now but... @@ -54,6 +59,8 @@ proc parseConfFile(filename: string) = uploadCommandFormat = e.value of "download": downloadCommandFormat = e.value + of "delete": + deleteCommandFormat = e.value else: quit("Unknown configuration: " & e.key) of cfgOption: @@ -70,7 +77,7 @@ proc run(cmd: string): auto = # Upload a file to the remote master blimpStore proc uploadFile(blimpFilename: string) = if remoteBlimpStore.isNil: - echo("Remote blimpstore not set in configuration file, not uploading content:\n\t" & blimpFilename) + echo("Remote blimpstore not set in configuration file, skipping uploading content:\n\t" & blimpFilename) return let errorCode = run(format(uploadCommandFormat, blimpFilename, remoteBlimpStore, blimpStore)) if errorCode != 0: @@ -84,6 +91,13 @@ proc downloadFile(blimpFilename: string) = if errorCode != 0: quit("Something went wrong downloading " & blimpFilename & " from " & remoteBlimpStore, 3) +# Delete a file from the remote master blimpStore +proc remoteDeleteFile(blimpFilename: string) = + if remoteBlimpStore.isNil: + return + let errorCode = run(format(deleteCommandFormat, blimpFilename, remoteBlimpStore, blimpStore)) + if errorCode != 0: + quit("Something went wrong deleting " & blimpFilename & " from " & remoteBlimpStore, 3) # Copy content to blimpStore, no upload yet. proc copyToBlimpStore(filename, blimpFilename: string) = @@ -92,40 +106,71 @@ proc copyToBlimpStore(filename, blimpFilename: string) = uploadFile(blimpFilename) # Copy content from blimpStore, and downloading first if needed -proc copyFromblimpStore(blimpFilename, filename: string) = +proc copyFromBlimpStore(blimpFilename, filename: string) = if not existsFile(blimpStore / blimpFilename): downloadFile(blimpFilename) copyFile(blimpStore / blimpFilename, filename) - +# Delete from blimpStore and remote. +proc deleteFromBlimpStore(blimpFilename, filename: string) = + if existsFile(blimpStore / blimpFilename): + removeFile(blimpStore / blimpFilename) + remoteDeleteFile(blimpFilename) + +# Pick out blimpFilename (filename & "-" & hash) +proc blimpFileName(filename: string): string= + var hashfile: File + if not open(hashfile, filename): + quit("Failed opening file: " & filename, 4) + let hashline = split(string(readLine(hashfile)), {':'}) + if hashline[0] == "hash": + result = hashline[1] + else: + result = nil + +# Get hash and compute blimpFilename +proc computeBlimpFilename(filename: string): string = + result = blimpFilename(filename) + if result.isNil: + var content: string + try: + content = readFile(filename) + except: + quit("Failed opening file: " & filename, 1) + let hash = getMD5(content) + result = filename & "-" & hash + # Copy original file to blimpStore and replace with hash stub in git. proc deflate(filename: string) = - var content: string - try: - content = readFile(filename) - except: - quit("Failed opening file: " & filename, 1) - if content[0..4] == "hash:": - quit("File is already deflated, ignored.", 5) - let hash = getMD5(content) - let blimpFilename = filename & "-" & hash + let blimpFilename = computeBlimpFilename(filename) copyToBlimpStore(filename, blimpFilename) writeFile(filename, "hash:" & blimpFilename) echo("\t" & filename & " deflated.") + +proc isInBlimpStore(filename: string): bool = + let blimpFilename = blimpFilename(filename) + if not blimpFilename.isNil: + return true # Parse out hash from hash stub and copy back original content from blimpStore. proc inflate(filename: string) = - var hashfile: File - if not open(hashfile, filename): - quit("Failed opening file: " & filename, 4) - let hashline = split(string(readLine(hashfile)), {':'}) - if hashline[0] == "hash": - let blimpfilename = hashline[1] - #removeFile(filename) - copyFromblimpStore(blimpfilename, filename) + let blimpFilename = blimpFilename(filename) + if blimpFilename.isNil: + echo("\t" & filename & " is not deflated, skipping.", 5) + else: + copyFromBlimpStore(blimpfilename, filename) + echo("\t" & filename & " inflated.") + +# Inflates file first (if deflated) and then removes current content for it, +# both locally and in remote. +proc remove(filename: string) = + var blimpFilename = blimpFilename(filename) + if not blimpFilename.isNil: + copyFromBlimpStore(blimpfilename, filename) else: - quit("\t" & filename & " is not deflated.", 5) - echo("\t" & filename & " inflated.") + blimpFilename = computeBlimpFilename(filename) + deleteFromBlimpStore(blimpfilename, filename) + echo("\t" & filename & " content removed from blimpstore locally and remotely.") # Find git root dir or fall back on current dir proc gitRoot(): string = @@ -141,8 +186,26 @@ proc gitRoot(): string = let help = """ blimp [options] -v,--verbose Verbosity - (string) (i)nflate or (d)eflate + (string) (d)eflate, (i)nflate, delete, (c)heck, (r)ecover (string...) One or more filepaths to inflate/deflate + + Edit ~/blimpstore/.blimp.conf or /.blimp.conf and set a proper + remote and also create ~/blimp.pass with the proper rsync password to use. + + Deflate is run before you add the big file to the index for committing. + Deflate will replace the file contents with a hash, and copy the + real content to ~/blimpstore, and if configured also upload it to + remote, using rsync. + + Inflate will bring back the original content by copying from + ~/blimpstore, and if its not there, first downloading from the remote. + Use this whenever you need to work/edit the big file - in order to get + its real content. + + Remove (no single character shortcut) will remove the file(s) content + both from the local ~/blimpstore and from the remote. This only removes + the current content version, not older versions. The file itself is first + inflated, if needed, and not deleted. This only "unblimps" the file. """ ################################ main ##################################### @@ -161,8 +224,10 @@ except: # Parse configuration files if they exist -parseConfFile(gitRoot() / ".blimp.conf") +# The one in gitroot overrides settings from the store. parseConfFile(blimpStore / ".blimp.conf") +parseConfFile(gitRoot() / ".blimp.conf") + # Using lapp to get args let args = parse(help) @@ -177,6 +242,9 @@ if command == "d" or command == "deflate": elif command == "i" or command == "inflate": for fn in filenames: inflate(fn.asString) +elif command == "remove": + for fn in filenames: + remove(fn.asString) else: quit("Unknown command, only (d)eflate or (i)inflate are valid.", 6) diff --git a/blimp.nimble b/blimp.nimble index 7ea455c..9c707e7 100644 --- a/blimp.nimble +++ b/blimp.nimble @@ -1,6 +1,6 @@ [Package] name = "blimp" -version = "0.1" +version = "0.2" author = "Göran Krampe" description = "Utility that helps with big files in git, very similar to git-fat, s3annnex etc." license = "MIT"