# this is a library to keep a shared repository with many forms of backups # atm, git and ikiwiki are required, but it should be reasonable to make other approaches # git saves history # ikiwiki connects git to online presentation and collaboration import os import shlex import shutil import subprocess import sys import tempfile import time import urllib __all__ = ["freenet", "bsv_metanet"] backend_path = os.path.normpath(os.path.join(os.path.dirname(__file__), '..')) root_path = os.path.normpath(os.path.join(backend_path, '..')) auth_path = os.path.join(backend_path, 'auth') dep_path = os.path.join(backend_path, 'dep') ikiwiki_setup_path = os.path.join(backend_path, 'ikiwiki.setup') html_path = os.path.normpath(os.path.join(root_path, 'html')) wiki_src_path = os.path.normpath(os.path.join(root_path, 'wiki')) git_path = os.path.normpath(os.path.join(root_path, '.git')) ssh_wrap_path = os.path.join(backend_path, "wrapped_ssh.sh") for auth_subpath, auth_subpaths, auth_subfiles in os.walk(auth_path): for auth_subfile in auth_subfiles: auth_subfile = os.path.join(auth_subpath, auth_subfile) if os.path.isfile(auth_subfile): os.chmod(auth_subfile, 0600) def url200(url): try: return urllib.urlopen(url).getcode() == 200 except IOError: return False class git: found = (os.system("git status --porcelain") == 0) if not found: print("ERROR: git not found") # TODO: download git, place in dep_path, and run from there exit(1) orig_dir = os.path.abspath(".") os.chdir(root_path) os.system("git submodule init") os.system("git submodule update") os.chdir(orig_dir) # http://stackoverflow.com/questions/1964470/whats-the-equivalent-of-use-commit-times-for-git # sets mtimes to last commit time, for ikiwiki 'last edited by' time filelist = set() for root, subdirs, files in os.walk(wiki_src_path): for file in files: filelist.add(os.path.relpath(os.path.join(root, file), root_path)) mtime = 0 gitobj = subprocess.Popen(shlex.split('git whatchanged --pretty=%at'), stdout=subprocess.PIPE) for line in gitobj.stdout: line = line.strip() if not line: continue if line.startswith(':'): file = line.split('\t')[-1] if file in filelist: filelist.remove(file) #print mtime, file os.utime(os.path.join(root_path,file), (mtime, mtime)) else: mtime = long(line) if not filelist: break def __init__(self, remote, push_remote = ""): self.remote = remote if push_remote == "": push_remote = remote self.push_remote = push_remote def pull(self): if self.remote == None: return False os.environ["GIT_SSH"] = ssh_wrap_path status = os.system("git pull %s master" % self.remote) del os.environ["GIT_SSH"] return status == 0 def push(self): if self.push_remote == None: return False os.environ["GIT_SSH"] = ssh_wrap_path status = os.system("git push %s master" % self.push_remote) del os.environ["GIT_SSH"] return status == 0 class ikiwiki: binary = "ikiwiki" found = (os.system("%s --version" % binary) == 0) if not found: binary = "%s/ikiwiki/ikiwiki.out" % dep_path os.environ['PERL5LIB'] += os.pathsep + os.path.join(dep_path, 'ikiwiki') found = (os.system("%s --version" % binary) == 0) if not found: print("ikiwiki not found: attempting to build") os.system("%s/build_ikiwiki.sh" % dep_path) found = (os.system("%s --version" % binary) == 0) if not found: print("WARNING: ikiwiki not found. did build fail on this platform?") synced = False def push(self): if not ikiwiki.found: return False if ikiwiki.synced: return True shutil.rmtree(html_path) os.system("git checkout -- '%s'" % html_path) status = os.system("%s --setup '%s' '%s' '%s'" % (ikiwiki.binary, ikiwiki_setup_path, wiki_src_path, html_path)) ikiwiki.synced = (status == 0) if ikiwiki.synced: os.system('git repack -d') os.system('git prune-packed') os.system('git update-server-info') # /.git folder access can be seen as security access problem by web hosts, so alias to /git with open(os.path.join(html_path, ".git"), "w") as f: f.write("gitdir: ./git") html_git_path = os.path.join(html_path, 'git') os.mkdir(html_git_path) for path in ['objects/info', 'objects/pack', 'refs', 'HEAD', 'info', 'packed-refs']: git_subpath = os.path.join(git_path, path) html_git_subpath = os.path.join(html_git_path, path) if os.path.isdir(git_subpath): shutil.copytree(git_subpath, html_git_subpath) else: shutil.copy2(git_subpath, html_git_subpath) return ikiwiki.synced class zeronet: found = 'ZERONETDIR' in os.environ if found: zn_datadir = os.path.join(os.environ['ZERONETDIR'], 'ZeroNet/data') zn_script = os.path.join(os.environ['ZERONETDIR'], 'ZeroNet.sh') found = (os.system("'%s' --version" % zn_script) == 0) if not found: print("WARNING: ZeroNet bundle not found in ZERONETDIR, public zeronet site won't be updated") running = url200('http://127.0.0.1:43110/1HeLLo4uzjaLetFx6NH3PMwFP3qbRbTf3D') if found and not running: proc = subprocess.Popen([zn_script], env={"DISPLAY":""}) while not running and proc.returncode == None: time.sleep(0.5) running = url200('http://127.0.0.1:43110/1HeLLo4uzjaLetFx6NH3PMwFP3qbRbTf3D') if not running: print("WARNING: ZeroNet not running, won't be used") def __init__(self, addr, key): self.addr = addr self.key = key def push(self): if not ikiwiki.synced or not zeronet.found: return False zitedir = os.path.join(zeronet.zn_datadir, self.addr) shutil.rmtree(zitedir, True) shutil.copytree(html_path, zitedir) status = os.system('%s siteSign %s %s' % (zeronet.zn_script, self.addr, self.key)) return status == 0 class ipfs: found = (os.system('ipfs version') == 0) if not found: print("WARNING: ipfs not found, ipfs content won't be updated") synced = False def __init__(self, configdir): self.configdir = os.path.join(auth_path, configdir) subprocess.Popen(['ipfs', '-c', self.configdir, 'daemon']) while self._config('Identity.PeerID') == None: time.sleep(0.5) self.peerid = self._config('Identity.PeerID') self.gateway = self._config('Addresses.Gateway').replace('/ip4','http:/').replace('/tcp/',':') def _config(self, key): proc = subprocess.Popen(['ipfs', '-c', self.configdir, 'config', key], stdout=subprocess.PIPE) for line in proc.stdout: return line[0:-1] def push(self): if not ikiwiki.synced or not ipfs.found: return False # add files to ipfs if not ipfs.synced: proc = subprocess.Popen(['ipfs', '-c', self.configdir, 'add', '-rH', html_path], stdout=subprocess.PIPE) for line in proc.stdout: print line[0:-1] ipfs.hash = line[6:52] ipfs.synced = True # publish hash to private key proc = subprocess.Popen(['ipfs', '-c', self.configdir, 'name', 'publish', ipfs.hash], stdout=subprocess.PIPE) for line in proc.stdout: print line[0:-1] self.peerid = line[13:59] status = proc.wait() return status == 0