"""Handle status as reported by servers with the 'report-status' capability.

def __init__(self):

self._done = False

self._pack_status = None

self._ref_status_ok = True

self._ref_statuses = []

def check(self):

"""Check if there were any errors and, if so, raise exceptions.

if self._pack_status not in ('unpack ok', None):

raise SendPackError(self._pack_status)

if not self._ref_status_ok:

ref_status = {}

ok = set()

for status in self._ref_statuses:

if ' ' not in status:

# malformed response, move on to the next one

continue

status, ref = status.split(' ', 1)

if status == 'ng':

if ' ' in ref:

ref, status = ref.split(' ', 1)

else:

ok.add(ref)

ref_status[ref] = status

raise UpdateRefsError('%s failed to update' %

', '.join([ref for ref in ref_status

if ref not in ok]),

ref_status=ref_status)

def handle_packet(self, pkt):

"""Handle a packet.

if self._done:

raise GitProtocolError("received more data after status report")

if pkt is None:

self._done = True

return

if self._pack_status is None:

self._pack_status = pkt.strip()

else:

ref_status = pkt.strip()

self._ref_statuses.append(ref_status)

if not ref_status.startswith('ok '):

"""Git smart server client.

def __init__(self, thin_packs=True, report_activity=None):

"""Create a new GitClient instance.

self._report_activity = report_activity

self._report_status_parser = None

self._fetch_capabilities = set(FETCH_CAPABILITIES)

self._send_capabilities = set(SEND_CAPABILITIES)

if not thin_packs:

self._fetch_capabilities.remove('thin-pack')

def _read_refs(self, proto):

server_capabilities = None

refs = {}

# Receive refs from server

for pkt in proto.read_pkt_seq():

(sha, ref) = pkt.rstrip('\n').split(' ', 1)

if sha == 'ERR':

raise GitProtocolError(ref)

if server_capabilities is None:

(ref, server_capabilities) = extract_capabilities(ref)

refs[ref] = sha

if len(refs) == 0:

return None, set([])

return refs, set(server_capabilities or [])

def send_pack(self, path, determine_wants, generate_pack_contents,

progress=None):

"""Upload a pack to a remote repository.

raise NotImplementedError(self.send_pack)

def fetch(self, path, target, determine_wants=None, progress=None):

"""Fetch into a target repository.

if determine_wants is None:

determine_wants = target.object_store.determine_wants_all

f, commit = target.object_store.add_pack()

result = self.fetch_pack(path, determine_wants,

target.get_graph_walker(), f.write, progress)

commit()

return result

def fetch_pack(self, path, determine_wants, graph_walker, pack_data,

progress=None):

"""Retrieve a pack from a git smart server.

raise NotImplementedError(self.fetch_pack)

def _parse_status_report(self, proto):

unpack = proto.read_pkt_line().strip()

if unpack != 'unpack ok':

st = True

# flush remaining error data

while st is not None:

st = proto.read_pkt_line()

raise SendPackError(unpack)

statuses = []

errs = False

ref_status = proto.read_pkt_line()

while ref_status:

ref_status = ref_status.strip()

statuses.append(ref_status)

if not ref_status.startswith('ok '):

errs = True

ref_status = proto.read_pkt_line()

if errs:

ref_status = {}

ok = set()

for status in statuses:

if ' ' not in status:

# malformed response, move on to the next one

continue

status, ref = status.split(' ', 1)

if status == 'ng':

if ' ' in ref:

ref, status = ref.split(' ', 1)

else:

ok.add(ref)

ref_status[ref] = status

raise UpdateRefsError('%s failed to update' %

', '.join([ref for ref in ref_status

if ref not in ok]),

ref_status=ref_status)

def _read_side_band64k_data(self, proto, channel_callbacks):

"""Read per-channel data.

for pkt in proto.read_pkt_seq():

channel = ord(pkt[0])

pkt = pkt[1:]

try:

cb = channel_callbacks[channel]

except KeyError:

raise AssertionError('Invalid sideband channel %d' % channel)

else:

if cb is not None:

cb(pkt)

def _handle_receive_pack_head(self, proto, capabilities, old_refs, new_refs):

"""Handle the head of a 'git-receive-pack' request.

want = []

have = [x for x in old_refs.values() if not x == ZERO_SHA]

sent_capabilities = False

for refname in set(new_refs.keys() + old_refs.keys()):

old_sha1 = old_refs.get(refname, ZERO_SHA)

new_sha1 = new_refs.get(refname, ZERO_SHA)

if old_sha1 != new_sha1:

if sent_capabilities:

proto.write_pkt_line('%s %s %s' % (old_sha1, new_sha1,

refname))

else:

proto.write_pkt_line(

'%s %s %s\0%s' % (old_sha1, new_sha1, refname,

' '.join(capabilities)))

sent_capabilities = True

if new_sha1 not in have and new_sha1 != ZERO_SHA:

want.append(new_sha1)

proto.write_pkt_line(None)

return (have, want)

def _handle_receive_pack_tail(self, proto, capabilities, progress=None):

"""Handle the tail of a 'git-receive-pack' request.

if "side-band-64k" in capabilities:

if progress is None:

progress = lambda x: None

channel_callbacks = { 2: progress }

if 'report-status' in capabilities:

channel_callbacks[1] = PktLineParser(

self._report_status_parser.handle_packet).parse

self._read_side_band64k_data(proto, channel_callbacks)

else:

if 'report-status' in capabilities:

for pkt in proto.read_pkt_seq():

self._report_status_parser.handle_packet(pkt)

if self._report_status_parser is not None:

self._report_status_parser.check()

# wait for EOF before returning

data = proto.read()

if data:

raise SendPackError('Unexpected response %r' % data)

def _handle_upload_pack_head(self, proto, capabilities, graph_walker,

wants, can_read):

"""Handle the head of a 'git-upload-pack' request.

assert isinstance(wants, list) and type(wants[0]) == str

proto.write_pkt_line('want %s %s\n' % (

wants[0], ' '.join(capabilities)))

for want in wants[1:]:

proto.write_pkt_line('want %s\n' % want)

proto.write_pkt_line(None)

have = graph_walker.next()

while have:

proto.write_pkt_line('have %s\n' % have)

if can_read():

pkt = proto.read_pkt_line()

parts = pkt.rstrip('\n').split(' ')

if parts[0] == 'ACK':

graph_walker.ack(parts[1])

if parts[2] in ('continue', 'common'):

pass

elif parts[2] == 'ready':

break

else:

raise AssertionError(

"%s not in ('continue', 'ready', 'common)" %

parts[2])

have = graph_walker.next()

proto.write_pkt_line('done\n')

def _handle_upload_pack_tail(self, proto, capabilities, graph_walker,

pack_data, progress=None, rbufsize=_RBUFSIZE):

"""Handle the tail of a 'git-upload-pack' request.

pkt = proto.read_pkt_line()

while pkt:

parts = pkt.rstrip('\n').split(' ')

if parts[0] == 'ACK':

graph_walker.ack(pkt.split(' ')[1])

if len(parts) < 3 or parts[2] not in (

'ready', 'continue', 'common'):

break

pkt = proto.read_pkt_line()

if "side-band-64k" in capabilities:

if progress is None:

# Just ignore progress data

progress = lambda x: None

self._read_side_band64k_data(proto, {1: pack_data, 2: progress})

# wait for EOF before returning

data = proto.read()

if data:

raise Exception('Unexpected response %r' % data)

else:

while True:

data = proto.read(rbufsize)

if data == "":

break

"""Traditional Git client."""

def _connect(self, cmd, path):

"""Create a connection to the server.

raise NotImplementedError()

def send_pack(self, path, determine_wants, generate_pack_contents,

progress=None):

"""Upload a pack to a remote repository.

proto, unused_can_read = self._connect('receive-pack', path)

old_refs, server_capabilities = self._read_refs(proto)

negotiated_capabilities = self._send_capabilities & server_capabilities

if 'report-status' in negotiated_capabilities:

self._report_status_parser = ReportStatusParser()

report_status_parser = self._report_status_parser

try:

new_refs = orig_new_refs = determine_wants(dict(old_refs))

except:

proto.write_pkt_line(None)

raise

if not 'delete-refs' in server_capabilities:

# Server does not support deletions. Fail later.

def remove_del(pair):

if pair[1] == ZERO_SHA:

if 'report-status' in negotiated_capabilities:

report_status_parser._ref_statuses.append(

'ng %s remote does not support deleting refs'

% pair[1])

report_status_parser._ref_status_ok = False

return False

else:

return True

new_refs = dict(

filter(

remove_del,

[(ref, sha) for ref, sha in new_refs.iteritems()]))

if new_refs is None:

proto.write_pkt_line(None)

return old_refs

if len(new_refs) == 0 and len(orig_new_refs):

# NOOP - Original new refs filtered out by policy

proto.write_pkt_line(None)

if self._report_status_parser is not None:

self._report_status_parser.check()

return old_refs

(have, want) = self._handle_receive_pack_head(proto,

negotiated_capabilities, old_refs, new_refs)

if not want and old_refs == new_refs:

return new_refs

objects = generate_pack_contents(have, want)

if len(objects) > 0:

entries, sha = write_pack_objects(proto.write_file(), objects)

elif len(set(new_refs.values()) - set([ZERO_SHA])) > 0:

# Check for valid create/update refs

filtered_new_refs = \

dict([(ref, sha) for ref, sha in new_refs.iteritems()

if sha != ZERO_SHA])

if len(set(filtered_new_refs.iteritems()) -

set(old_refs.iteritems())) > 0:

entries, sha = write_pack_objects(proto.write_file(), objects)

self._handle_receive_pack_tail(proto, negotiated_capabilities,

progress)

return new_refs

def fetch_pack(self, path, determine_wants, graph_walker, pack_data,

progress=None):

"""Retrieve a pack from a git smart server.

proto, can_read = self._connect('upload-pack', path)

refs, server_capabilities = self._read_refs(proto)

negotiated_capabilities = self._fetch_capabilities & server_capabilities

if refs is None:

proto.write_pkt_line(None)

return refs

try:

wants = determine_wants(refs)

except:

proto.write_pkt_line(None)

raise

if wants is not None:

wants = [cid for cid in wants if cid != ZERO_SHA]

if not wants:

proto.write_pkt_line(None)

return refs

self._handle_upload_pack_head(proto, negotiated_capabilities,

graph_walker, wants, can_read)

self._handle_upload_pack_tail(proto, negotiated_capabilities,

graph_walker, pack_data, progress)

return refs

def archive(self, path, committish, write_data, progress=None):

proto, can_read = self._connect('upload-archive', path)

proto.write_pkt_line("argument %s" % committish)

proto.write_pkt_line(None)

pkt = proto.read_pkt_line()

if pkt == "NACK\n":

return

elif pkt == "ACK\n":

pass

elif pkt.startswith("ERR "):

raise GitProtocolError(pkt[4:].rstrip("\n"))

else:

raise AssertionError("invalid response %r" % pkt)

ret = proto.read_pkt_line()

if ret is not None:

raise AssertionError("expected pkt tail")

"""A Git Client that works over TCP directly (i.e. git://)."""

def __init__(self, host, port=None, *args, **kwargs):

if port is None:

port = TCP_GIT_PORT

self._host = host

self._port = port

TraditionalGitClient.__init__(self, *args, **kwargs)

def _connect(self, cmd, path):

sockaddrs = socket.getaddrinfo(self._host, self._port,

socket.AF_UNSPEC, socket.SOCK_STREAM)

s = None

err = socket.error("no address found for %s" % self._host)

for (family, socktype, proto, canonname, sockaddr) in sockaddrs:

s = socket.socket(family, socktype, proto)

s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)

try:

s.connect(sockaddr)

break

except socket.error, err:

if s is not None:

s.close()

s = None

if s is None:

raise err

# -1 means system default buffering

rfile = s.makefile('rb', -1)

# 0 means unbuffered

wfile = s.makefile('wb', 0)

proto = Protocol(rfile.read, wfile.write,

report_activity=self._report_activity)

if path.startswith("/~"):

path = path[1:]

proto.send_cmd('git-%s' % cmd, path, 'host=%s' % self._host)

"""A socket-like object that talks to a subprocess via pipes."""

def __init__(self, proc):

self.proc = proc

self.read = proc.stdout.read

self.write = proc.stdin.write

def can_read(self):

if subprocess.mswindows:

from msvcrt import get_osfhandle

from win32pipe import PeekNamedPipe

handle = get_osfhandle(self.proc.stdout.fileno())

return PeekNamedPipe(handle, 0)[2] != 0

else:

return _fileno_can_read(self.proc.stdout.fileno())

def close(self):

self.proc.stdin.close()

self.proc.stdout.close()

"""Git client that talks to a server using a subprocess."""

def __init__(self, *args, **kwargs):

self._connection = None

self._stderr = None

self._stderr = kwargs.get('stderr')

if 'stderr' in kwargs:

del kwargs['stderr']

TraditionalGitClient.__init__(self, *args, **kwargs)

def _connect(self, service, path):

import subprocess

argv = ['git', service, path]

p = SubprocessWrapper(

subprocess.Popen(argv, bufsize=0, stdin=subprocess.PIPE,

stdout=subprocess.PIPE,

stderr=self._stderr))

return Protocol(p.read, p.write,

def __init__(self, write_function=None):

self.write_function = write_function or (lambda x: None)

def write(self, data, *args, **kwargs):

"""A client side SSH implementation."""

def connect_ssh(self, host, command, username=None, port=None):

"""Connect to an SSH server.

"""SSH vendor that shells out to the local 'ssh' command."""

def connect_ssh(self, host, command, username=None, port=None, progress_stderr=None, **kwargs):

import subprocess

#FIXME: This has no way to deal with passwords..

args = ['ssh', '-x']

if port is not None:

args.extend(['-p', str(port)])

if username is not None:

host = '%s@%s' % (username, host)

args.append(host)

stderr_stream = WritableStream(progress_stderr)

proc = subprocess.Popen(args + command,

stdin=subprocess.PIPE,

stdout=subprocess.PIPE,

stderr=stderr_stream)

STDERR_READ_N = 2048 # 2k

def __init__(self, client, channel, progress_stderr=None):

self.client = client

self.channel = channel

self.progress_stderr = progress_stderr

self.should_monitor = bool(progress_stderr) or True

self.monitor_thread = None

self.stderr = ''

# Channel must block

self.channel.setblocking(True)

# Start

if self.should_monitor:

self.monitor_thread = threading.Thread(target=self.monitor_stderr)

self.monitor_thread.start()

def monitor_stderr(self):

while self.should_monitor:

# Block and read

data = self.read_stderr(self.STDERR_READ_N)

# Socket closed

if not data:

self.should_monitor = False

break

# Emit data

if self.progress_stderr:

self.progress_stderr(data)

# Append to buffer

self.stderr += data

def stop_monitoring(self):

# Stop StdErr thread

if self.should_monitor:

self.should_monitor = False

self.monitor_thread.join()

# Get left over data

data = self.channel.in_stderr_buffer.empty()

self.stderr += data

def can_read(self):

return self.channel.recv_ready()

def write(self, data):

return self.channel.sendall(data)

def read_stderr(self, n):

return self.channel.recv_stderr(n)

def read(self, n=None):

data = self.channel.recv(n)

data_len = len(data)

# Closed socket

if not data:

return

# Read more if needed

if n and data_len < n:

diff_len = n - data_len

return data + self.read(diff_len)

return data

def close(self):

self.channel.close()

self.stop_monitoring()

def __del__(self):

SSH_DEFAULTS = {

'port': 22,

'look_for_keys': False,

'allow_agent': False,

}

def connect_ssh(self, host, command, progress_stderr=None, **kwargs):

client = paramiko.SSHClient()

# Set defaults for non existant keys

# as well as keys whose value is None

for k, v in self.SSH_DEFAULTS.items():

if kwargs.get(k) is None:

kwargs[k] = v

policy = paramiko.client.MissingHostKeyPolicy()

client.set_missing_host_key_policy(policy)

client.connect(host, **kwargs)

# Open SSH session

channel = client.get_transport().open_session()

# Run commands

apply(channel.exec_command, command)

SSH_KWARGS_KEYS = (

'username',

'password',

'port',

'pkey',

'look_for_keys',

'allow_agent',

'key_filename',

'progress_stderr'

)

def __init__(self, host, *args, **kwargs):

self.host = host

# Extra SSH kwargs

for key in self.SSH_KWARGS_KEYS:

# Remove value from kwargs

# So that we don't pass it to the TraditionalGitClient constructor

# AND

# Set all SSH kwargs as attributes to self (dulwich needs this)

# default those not in **kwargs to None

setattr(self, key, kwargs.pop(key, None))

# Contstruct traditional Git Client

TraditionalGitClient.__init__(self, *args, **kwargs)

self.alternative_paths = {}

def _ssh_kwargs(self):

return dict(

(key, getattr(self, key, None))

for key in self.SSH_KWARGS_KEYS

if not getattr(self, key, None) is None

)

def _get_cmd_path(self, cmd):

return self.alternative_paths.get(cmd, 'git-%s' % cmd)

def _connect(self, cmd, path):

if path.startswith("/~"):

path = path[1:]

command = ["%s '%s'" % (self._get_cmd_path(cmd), path)]

con = get_ssh_vendor().connect_ssh(self.host, command, **self._ssh_kwargs())

return (Protocol(con.read, con.write, report_activity=self._report_activity),

def __init__(self, base_url, dumb=None, *args, **kwargs):

self.base_url = base_url.rstrip("/") + "/"

self.dumb = dumb

GitClient.__init__(self, *args, **kwargs)

def _get_url(self, path):

return urlparse.urljoin(self.base_url, path).rstrip("/") + "/"

def _http_request(self, url, headers={}, data=None):

req = urllib2.Request(url, headers=headers, data=data)

try:

resp = self._perform(req)

except urllib2.HTTPError as e:

if e.code == 404:

raise NotGitRepository()

if e.code != 200:

raise GitProtocolError("unexpected http response %d" % e.code)

return resp

def _perform(self, req):

"""Perform a HTTP request.

return urllib2.urlopen(req)

def _discover_references(self, service, url):

assert url[-1] == "/"

url = urlparse.urljoin(url, "info/refs")

headers = {}

if self.dumb != False:

url += "?service=%s" % service

headers["Content-Type"] = "application/x-%s-request" % service

resp = self._http_request(url, headers)

self.dumb = (not resp.info().gettype().startswith("application/x-git-"))

proto = Protocol(resp.read, None)

if not self.dumb:

# The first line should mention the service

pkts = list(proto.read_pkt_seq())

if pkts != [('# service=%s\n' % service)]:

raise GitProtocolError(

"unexpected first line %r from smart server" % pkts)

return self._read_refs(proto)

def _smart_request(self, service, url, data):

assert url[-1] == "/"

url = urlparse.urljoin(url, service)

headers = {"Content-Type": "application/x-%s-request" % service}

resp = self._http_request(url, headers, data)

if resp.info().gettype() != ("application/x-%s-result" % service):

raise GitProtocolError("Invalid content-type from server: %s"

% resp.info().gettype())

return resp

def send_pack(self, path, determine_wants, generate_pack_contents,

progress=None):

"""Upload a pack to a remote repository.

url = self._get_url(path)

old_refs, server_capabilities = self._discover_references(

"git-receive-pack", url)

negotiated_capabilities = self._send_capabilities & server_capabilities

if 'report-status' in negotiated_capabilities:

self._report_status_parser = ReportStatusParser()

new_refs = determine_wants(dict(old_refs))

if new_refs is None:

return old_refs

if self.dumb:

raise NotImplementedError(self.fetch_pack)

req_data = StringIO()

req_proto = Protocol(None, req_data.write)

(have, want) = self._handle_receive_pack_head(

req_proto, negotiated_capabilities, old_refs, new_refs)

if not want and old_refs == new_refs:

return new_refs

objects = generate_pack_contents(have, want)

if len(objects) > 0:

entries, sha = write_pack_objects(req_proto.write_file(), objects)

resp = self._smart_request("git-receive-pack", url,

data=req_data.getvalue())

resp_proto = Protocol(resp.read, None)

self._handle_receive_pack_tail(resp_proto, negotiated_capabilities,

progress)

return new_refs

def fetch_pack(self, path, determine_wants, graph_walker, pack_data,

progress=None):

"""Retrieve a pack from a git smart server.

url = self._get_url(path)

refs, server_capabilities = self._discover_references(

"git-upload-pack", url)

negotiated_capabilities = self._fetch_capabilities & server_capabilities

wants = determine_wants(refs)

if wants is not None:

wants = [cid for cid in wants if cid != ZERO_SHA]

if not wants:

return refs

if self.dumb:

raise NotImplementedError(self.send_pack)

req_data = StringIO()

req_proto = Protocol(None, req_data.write)

self._handle_upload_pack_head(req_proto,

negotiated_capabilities, graph_walker, wants,

lambda: False)

resp = self._smart_request("git-upload-pack", url,

data=req_data.getvalue())

resp_proto = Protocol(resp.read, None)

self._handle_upload_pack_tail(resp_proto, negotiated_capabilities,

graph_walker, pack_data, progress)

"""Obtain a git client from a URI or path.

# Parse URL Scheme

parsed = urlparse.urlparse(uri)

# Extract port

# if not port in url port object will be None

port = parsed.port