import md5, os, osproc, parseopt2, strutils, parsecfg, streams # This is a little utility program for handling large files # in git repositories. Its inspired by git-fat and s3annex # but doesn't rely on S3 for storage. # # Use "fat d mybigfile" to deflate it before commit. # Use "fat i mybigfile" to inflate it back to original size. # # When deflated the file only has an md5sum string inside it. # # The file is copied over into: # /fatstore/- var fatstore, remoteFatstore: string # Load fatstore.conf file, overkill for now but... proc parseConfFile(filename: string) = echo filename var f = newFileStream(filename, fmRead) if f != nil: var p: CfgParser open(p, f, filename) while true: var e = next(p) case e.kind of cfgEof: break of cfgSectionStart: continue # Ignore of cfgKeyValuePair: if e.key == "remote": remoteFatstore = e.value else: quit("Unknown configuration: " & e.key) of cfgOption: quit("Unknown configuration: " & e.key) of cfgError: quit("Parsing " & filename & ": " & e.msg) close(p) # Upload a file to the remote master fatstore proc uploadFile(fatfilename: string) = if remoteFatstore.isNil: echo("Remote fatstore not set in configuration file, not uploading content:\n\t" & fatfilename) return let errorCode = execCmd("rsync -a " & fatstore / fatfilename & " " & remoteFatstore) if errorCode != 0: quit("Something went wrong uploading content to " & remoteFatstore, 2) # Download a file to the remote master fatstore proc downloadFile(fatfilename: string) = if remoteFatstore.isNil: quit("Remote fatstore not set in configuration file, can not download content:\n\t" & fatfilename) let errorCode = execCmd("rsync -a " & remoteFatstore / fatfilename & " " & fatstore / "") if errorCode != 0: quit("Something went wrong downloading " & fatfilename & " from " & remoteFatstore, 3) # Copy content to fatstore, no upload yet. proc copyToFatstore(filename, fatfilename: string) = if not existsFile(fatstore / fatfilename): copyFile(filename, fatstore / fatfilename) uploadFile(fatfilename) # Copy content from fatstore, and downloading first if needed proc copyFromFatstore(fatfilename, filename: string) = if not existsFile(fatstore / fatfilename): downloadFile(fatfilename) copyFile(fatstore / fatfilename, filename) # Copy original file to fatstore and replace with hash stub in git. proc deflate(filename: string) = let content = readFile(filename) if content[0..4] == "hash:": quit("File is already deflated, ignored.", 5) let hash = getMD5(content) let fatfilename = filename & "-" & hash copyToFatstore(filename, fatfilename) writeFile(filename, "hash:" & fatfilename) # Parse out hash from hash stub and copy back original content from fatstore. proc inflate(filename: string) = var hashfile: File if not open(hashfile, filename): quit("Could not open file: " & filename, 4) let hashline = split(string(readLine(hashfile)), {':'}) if hashline[0] == "hash": let fatfilename = hashline[1] #removeFile(filename) copyFromFatstore(fatfilename, filename) else: quit("File is not a fat file.", 5) ################################ main ##################################### # Hardwired to "fatstore" directory in home dir. fatstore = getHomeDir() / "fatstore" # Make sure we have the dir, or create it. try: if not existsDir(fatstore): createDir(fatstore) except: quit("Could not create " & fatstore & " directory.", 1) # Parse configuration if it exists parseConfFile(fatstore / "fatstore.conf") # Only a command and a path as argument let argv = commandLineParams() let command = argv[0] let filename = argv[1] # Do the deed if command == "d" or command == "deflate": deflate(filename) elif command == "i" or command == "inflate": inflate(filename) else: quit("Unknown command, only (d)eflate or (i)inflate are valid.", 6) # All good quit(0)