Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add initial support for Git protocol v2 #1244

Merged
merged 12 commits into from
Jun 28, 2024
Merged

add initial support for Git protocol v2 #1244

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
2b975ec
add initial support for Git protocol v2
stspdotname Jun 25, 2024
e543e98
keep dulwich pull command line similar to the Git CLI; suggested by J…
stspdotname Jun 25, 2024
26344f8
use None to represent unspecified optional parameters; suggested by J…
stspdotname Jun 25, 2024
325de7d
make the Git protocol version configurable from the command line
stspdotname Jun 25, 2024
2400933
make Git protocol v2 work over SSH connections
stspdotname Jun 25, 2024
2647c58
use dict() instead of deepcopy()
stspdotname Jun 25, 2024
65220bb
emulate Git's behaviour when a server does not support object filtering
stspdotname Jun 25, 2024
343ed68
set the default git protocol version via global constants
stspdotname Jun 25, 2024
5c1a5ec
don't hard-code protocol v2 and tweak version comparisons to be futur…
stspdotname Jun 25, 2024
5b2f74f
skip test_fetch_pack_no_side_band_64k if git protocol v2 is used
stspdotname Jun 25, 2024
0e18ede
run client compat tests with git protocol version 0, too
stspdotname Jun 25, 2024
f270f0c
blind attempt at fixing the unexpected-success "failures" on windows CI
stspdotname Jun 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
make the Git protocol version configurable from the command line
  • Loading branch information
@stspdotname
stspdotname committed Jun 25, 2024
commit 325de7d3731d19a8290c08a540a09024b4b1c7fd
6 changes: 6 additions & 0 deletions dulwich/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,9 @@ def run(self, args):
type=str,
help="git-rev-list-style object filter",
)
parser.add_option(
"--protocol", dest="protocol", type=int, help="Git protocol version to use"
)
options, args = parser.parse_args(args)

if args == []:
Expand All @@ -297,6 +300,7 @@ def run(self, args):
branch=options.branch,
refspec=options.refspec,
filter_spec=options.filter_spec,
protocol_version=options.protocol,
)
except GitProtocolError as e:
print(f"{e}")
Expand Down Expand Up @@ -605,12 +609,14 @@ def run(self, args):
parser.add_argument("from_location", type=str)
parser.add_argument("refspec", type=str, nargs="*")
parser.add_argument("--filter", type=str, nargs=1)
parser.add_argument("--protocol", type=int, nargs=1)
args = parser.parse_args(args)
porcelain.pull(
".",
args.from_location or None,
args.refspec or None,
filter_spec=args.filter,
protocol_version=args.protocol_version or None,
)


Expand Down
128 changes: 109 additions & 19 deletions dulwich/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@
extract_capability_names,
parse_capability,
pkt_line,
GIT_PROTOCOL_VERSIONS,
)
from .refs import PEELED_TAG_SUFFIX, _import_remote_refs, read_info_refs
from .repo import Repo
Expand Down Expand Up @@ -545,7 +546,7 @@ def _handle_upload_pack_head(
wants,
can_read,
depth,
protocol_version=None,
protocol_version,
):
"""Handle the head of a 'git-upload-pack' request.

Expand All @@ -557,7 +558,7 @@ def _handle_upload_pack_head(
can_read: function that returns a boolean that indicates
whether there is extra graph data to read on proto
depth: Depth for request
protocol_version: desired Git protocol version; defaults to v0
protocol_version: Neogiated Git protocol version.
"""
assert isinstance(wants, list) and isinstance(wants[0], bytes)
wantcmd = COMMAND_WANT + b" " + wants[0]
Expand Down Expand Up @@ -639,6 +640,7 @@ def _handle_upload_pack_tail(
pack_data: Function to call with pack data
progress: Optional progress reporting function
rbufsize: Read buffer size
protocol_version: Neogiated Git protocol version.
"""
pkt = proto.read_pkt_line()
while pkt:
Expand Down Expand Up @@ -782,6 +784,7 @@ def clone(
depth=None,
ref_prefix=[],
filter_spec=None,
protocol_version: Optional[int] = None,
) -> Repo:
"""Clone a repository."""
from .refs import _set_default_branch, _set_head, _set_origin_head
Expand Down Expand Up @@ -827,6 +830,7 @@ def clone(
depth=depth,
ref_prefix=ref_prefix,
filter_spec=filter_spec,
protocol_version=protocol_version,
)
if origin is not None:
_import_remote_refs(
Expand Down Expand Up @@ -878,6 +882,7 @@ def fetch(
depth: Optional[int] = None,
ref_prefix: Optional[List[bytes]] = [],
filter_spec: Optional[bytes] = None,
protocol_version: Optional[int] = None,
) -> FetchPackResult:
"""Fetch into a target repository.

Expand All @@ -898,6 +903,8 @@ def fetch(
filter_spec: A git-rev-list-style object filter spec, as bytestring.
Only used if the server supports the Git protocol-v2 'filter'
feature, and ignored otherwise.
protocol_version: Desired Git protocol version. By default the highest
mutually supported protocol version will be used.

Returns:
Dictionary with all remote refs (not just those fetched)
Expand Down Expand Up @@ -935,6 +942,7 @@ def abort():
depth=depth,
ref_prefix=ref_prefix,
filter_spec=filter_spec,
protocol_version=protocol_version,
)
except BaseException:
abort()
Expand All @@ -955,6 +963,7 @@ def fetch_pack(
depth: Optional[int] = None,
ref_prefix=[],
filter_spec=None,
protocol_version: Optional[int] = None,
):
"""Retrieve a pack from a git smart server.

Expand All @@ -976,6 +985,8 @@ def fetch_pack(
filter_spec: A git-rev-list-style object filter spec, as bytestring.
Only used if the server supports the Git protocol-v2 'filter'
feature, and ignored otherwise.
protocol_version: Desired Git protocol version. By default the highest
mutually supported protocol version will be used.

Returns:
FetchPackResult object
Expand Down Expand Up @@ -1133,7 +1144,7 @@ def __init__(self, path_encoding=DEFAULT_ENCODING, **kwargs) -> None:
self._remote_path_encoding = path_encoding
super().__init__(**kwargs)

async def _connect(self, cmd, path):
async def _connect(self, cmd, path, protocol_version=None):
"""Create a connection to the server.

This method is abstract - concrete implementations should
Expand All @@ -1145,6 +1156,8 @@ async def _connect(self, cmd, path):
Args:
cmd: The git service name to which we should connect.
path: The path we should pass to the service. (as bytestirng)
protocol_version: Desired Git protocol version. By default the highest
mutually supported protocol version will be used.
"""
raise NotImplementedError

Expand Down Expand Up @@ -1252,6 +1265,7 @@ def fetch_pack(
depth=None,
ref_prefix=[],
filter_spec=None,
protocol_version: Optional[int] = None,
):
"""Retrieve a pack from a git smart server.

Expand All @@ -1273,13 +1287,30 @@ def fetch_pack(
filter_spec: A git-rev-list-style object filter spec, as bytestring.
Only used if the server supports the Git protocol-v2 'filter'
feature, and ignored otherwise.
protocol_version: Desired Git protocol version. By default the highest
mutually supported protocol version will be used.

Returns:
FetchPackResult object

"""
proto, can_read, stderr = self._connect(b"upload-pack", path)
self.protocol_version = negotiate_protocol_version(proto)
if (
protocol_version is not None
and protocol_version not in GIT_PROTOCOL_VERSIONS
):
raise ValueError("unknown Git protocol version %d" % protocol_version)
proto, can_read, stderr = self._connect(b"upload-pack", path, protocol_version)
server_protocol_version = negotiate_protocol_version(proto)
if server_protocol_version not in GIT_PROTOCOL_VERSIONS:
raise ValueError(
"unknown Git protocol version %d used by server"
% server_protocol_version
)
if protocol_version and server_protocol_version > protocol_version:
raise ValueError(
"bad Git protocol version %d used by server" % server_protocol_version
)
self.protocol_version = server_protocol_version
with proto:
try:
if self.protocol_version == 2:
Expand Down Expand Up @@ -1352,11 +1383,26 @@ def fetch_pack(
)
return FetchPackResult(refs, symrefs, agent, new_shallow, new_unshallow)

def get_refs(self, path):
def get_refs(self, path, protocol_version=None):
"""Retrieve the current refs from a git smart server."""
# stock `git ls-remote` uses upload-pack
proto, _, stderr = self._connect(b"upload-pack", path)
self.protocol_version = negotiate_protocol_version(proto)
if (
protocol_version is not None
and protocol_version not in GIT_PROTOCOL_VERSIONS
):
raise ValueError("unknown Git protocol version %d" % protocol_version)
proto, _, stderr = self._connect(b"upload-pack", path, protocol_version)
server_protocol_version = negotiate_protocol_version(proto)
if server_protocol_version not in GIT_PROTOCOL_VERSIONS:
raise ValueError(
"unknown Git protocol version %d used by server"
% server_protocol_version
)
if protocol_version and server_protocol_version > protocol_version:
raise ValueError(
"bad Git protocol version %d used by server" % server_protocol_version
)
self.protocol_version = server_protocol_version
if self.protocol_version == 2:
server_capabilities = read_server_capabilities(proto.read_pkt_seq())
proto.write_pkt_line(b"command=ls-refs\n")
Expand Down Expand Up @@ -1443,7 +1489,7 @@ def get_url(self, path):
netloc += ":%d" % self._port
return urlunsplit(("git", netloc, path, "", ""))

def _connect(self, cmd, path):
def _connect(self, cmd, path, protocol_version=None):
if not isinstance(cmd, bytes):
raise TypeError(cmd)
if not isinstance(path, bytes):
Expand Down Expand Up @@ -1485,15 +1531,21 @@ def close():
if path.startswith(b"/~"):
path = path[1:]
if cmd == b"upload-pack":
self.protocol_version = 2
if protocol_version is None:
self.protocol_version = 2
else:
self.protocol_version = protocol_version
else:
self.protocol_version = 0

if cmd == b"upload-pack" and self.protocol_version == 2:
# Git protocol version advertisement is hidden behind two NUL bytes
# for compatibility with older Git server implementations, which
# would crash if something other than a "host=" header was found
# after the first NUL byte.
version_str = b"\0\0version=%d\0" % self.protocol_version
else:
version_str = b""
self.protocol_version = 0
# TODO(jelmer): Alternative to ascii?
proto.send_cmd(
b"git-" + cmd, path, b"host=" + self._host.encode("ascii") + version_str
Expand Down Expand Up @@ -1557,7 +1609,7 @@ def from_parsedurl(cls, parsedurl, **kwargs):

git_command = None

def _connect(self, service, path):
def _connect(self, service, path, protocol_version=None):
if not isinstance(service, bytes):
raise TypeError(service)
if isinstance(path, bytes):
Expand Down Expand Up @@ -1683,6 +1735,7 @@ def fetch(
depth=None,
ref_prefix=[],
filter_spec=None,
**kwargs,
):
"""Fetch into a target repository.

Expand Down Expand Up @@ -1727,6 +1780,7 @@ def fetch_pack(
depth=None,
ref_prefix: Optional[List[bytes]] = [],
filter_spec: Optional[bytes] = None,
protocol_version: Optional[int] = None,
) -> FetchPackResult:
"""Retrieve a pack from a local on-disk repository.

Expand Down Expand Up @@ -1807,6 +1861,8 @@ def run_command(
password: Optional ssh password for login or private key
key_filename: Optional path to private keyfile
ssh_command: Optional SSH command
protocol_version: Desired Git protocol version. By default the highest
mutually supported protocol version will be used.
"""
raise NotImplementedError(self.run_command)

Expand Down Expand Up @@ -1986,7 +2042,7 @@ def _get_cmd_path(self, cmd):
assert isinstance(cmd, bytes)
return cmd

def _connect(self, cmd, path):
def _connect(self, cmd, path, protocol_version=None):
if not isinstance(cmd, bytes):
raise TypeError(cmd)
if isinstance(path, bytes):
Expand Down Expand Up @@ -2214,7 +2270,12 @@ def _http_request(self, url, headers=None, data=None):
"""
raise NotImplementedError(self._http_request)

def _discover_references(self, service, base_url):
def _discover_references(self, service, base_url, protocol_version=None):
if (
protocol_version is not None
and protocol_version not in GIT_PROTOCOL_VERSIONS
):
raise ValueError("unknown Git protocol version %d" % protocol_version)
assert base_url[-1] == "/"
tail = "info/refs"
headers = {"Accept": "*/*"}
Expand All @@ -2226,8 +2287,12 @@ def _discover_references(self, service, base_url):
# we try: It responds with a Git-protocol-v1-style ref listing
# which lacks the "001f# service=git-receive-pack" marker.
if service == b"git-upload-pack":
self.protocol_version = 2
headers["Git-Protocol"] = "version=2"
if protocol_version is None:
self.protocol_version = 2
else:
self.protocol_version = protocol_version
if self.protocol_version == 2:
headers["Git-Protocol"] = "version=2"
else:
self.protocol_version = 0
url = urljoin(base_url, tail)
Expand Down Expand Up @@ -2261,7 +2326,18 @@ def begin_protocol_v2(proto):
return server_capabilities, resp, read, proto

proto = Protocol(read, None)
self.protocol_version = negotiate_protocol_version(proto)
server_protocol_version = negotiate_protocol_version(proto)
if server_protocol_version not in GIT_PROTOCOL_VERSIONS:
raise ValueError(
"unknown Git protocol version %d used by server"
% server_protocol_version
)
if protocol_version and server_protocol_version > protocol_version:
raise ValueError(
"bad Git protocol version %d used by server"
% server_protocol_version
)
self.protocol_version = server_protocol_version
if self.protocol_version == 2:
server_capabilities, resp, read, proto = begin_protocol_v2(proto)
else:
Expand All @@ -2278,7 +2354,18 @@ def begin_protocol_v2(proto):
)
# Github sends "version 2" after sending the service name.
# Try to negotiate protocol version 2 again.
self.protocol_version = negotiate_protocol_version(proto)
server_protocol_version = negotiate_protocol_version(proto)
if server_protocol_version not in GIT_PROTOCOL_VERSIONS:
raise ValueError(
"unknown Git protocol version %d used by server"
% server_protocol_version
)
if protocol_version and server_protocol_version > protocol_version:
raise ValueError(
"bad Git protocol version %d used by server"
% server_protocol_version
)
self.protocol_version = server_protocol_version
if self.protocol_version == 2:
server_capabilities, resp, read, proto = begin_protocol_v2(
proto
Expand Down Expand Up @@ -2393,6 +2480,7 @@ def fetch_pack(
depth=None,
ref_prefix=[],
filter_spec=None,
protocol_version: Optional[int] = None,
):
"""Retrieve a pack from a git smart server.

Expand All @@ -2412,14 +2500,16 @@ def fetch_pack(
filter_spec: A git-rev-list-style object filter spec, as bytestring.
Only used if the server supports the Git protocol-v2 'filter'
feature, and ignored otherwise.
protocol_version: Desired Git protocol version. By default the highest
mutually supported protocol version will be used.

Returns:
FetchPackResult object

"""
url = self._get_url(path)
refs, server_capabilities, url = self._discover_references(
b"git-upload-pack", url
b"git-upload-pack", url, protocol_version
)
(
negotiated_capabilities,
Expand Down
Loading