Skip to content

Commit

Permalink
Merge pull request #183 from radam9/master
Browse files Browse the repository at this point in the history 
fix UnicodeDecodeError when downloading non-json content
  • Loading branch information
@jph00
jph00 authored Aug 31, 2024
2 parents 07d5ef7 + 4fb1e5a commit 2093545
Show file tree
Hide file tree
Showing 3 changed files with 33 additions and 11 deletions.
21 changes: 16 additions & 5 deletions 00_core.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,16 @@
" params += [_mk_param(k, **_mk_sig_detls(v)) for k,v in anno_args.items()]\n",
" return Signature(params)\n",
"\n",
"def _decode_response(path: str) -> bool:\n",
" \"checks if a endpoint needs to have it's response from `fastcore.core.urlsend` decoded or just return json\"\n",
" needs_decode = (\n",
" \"/orgs/{org}/migrations/{migration_id}/archive\",\n",
" \"/repos/{owner}/{repo}/actions/artifacts/{artifact_id}/{archive_format}\",\n",
" \"/repos/{owner}/{repo}/tarball/{ref}\",\n",
" \"/repos/{owner}/{repo}/zipball/{ref}\",\n",
" )\n",
" return path not in needs_decode\n",
"\n",
"class _GhObj: pass"
]
},
Expand All @@ -99,11 +109,12 @@
"source": [
"#|export\n",
"class _GhVerb(_GhObj):\n",
" __slots__ = 'path,verb,tag,name,summary,url,route_ps,params,data,preview,client,__doc__'.split(',')\n",
" __slots__ = 'path,verb,tag,name,summary,url,route_ps,params,data,preview,client,decode,__doc__'.split(',')\n",
" def __init__(self, path, verb, oper, summary, url, params, data, preview, client, kwargs):\n",
" tag,*name = oper.split('/')\n",
" name = '__'.join(name)\n",
" name = name.replace('-','_')\n",
" decode = _decode_response(path)\n",
" path,_,_ = partial_format(path, **kwargs)\n",
" route_ps = stringfmt_names(path)\n",
" __doc__ = summary\n",
Expand All @@ -117,7 +128,7 @@
" for a,b in zip(args,flds): kwargs[b]=a\n",
" route_p,query_p,data_p = [{p:kwargs[p] for p in o if p in kwargs}\n",
" for o in (self.route_ps,self.params,d)]\n",
" return self.client(self.path, self.verb, headers=headers, route=route_p, query=query_p, data=data_p)\n",
" return self.client(self.path, self.verb, headers=headers, decode=self.decode, route=route_p, query=query_p, data=data_p)\n",
"\n",
" def __str__(self): return f'{self.tag}.{self.name}{signature(self)}\\n{self.doc_url}'\n",
" @property\n",
Expand Down Expand Up @@ -201,17 +212,17 @@
" self.debug,self.limit_cb,self.limit_rem = debug,limit_cb,5000\n",
" self.gh_host = gh_host or GH_HOST\n",
"\n",
" def __call__(self, path:str, verb:str=None, headers:dict=None, route:dict=None, query:dict=None, data=None, timeout=None):\n",
" def __call__(self, path:str, verb:str=None, headers:dict=None, route:dict=None, query:dict=None, data=None, timeout=None, decode=True):\n",
" \"Call a fully specified `path` using HTTP `verb`, passing arguments to `fastcore.core.urlsend`\"\n",
" if verb is None: verb = 'POST' if data else 'GET'\n",
" headers = {**self.headers,**(headers or {})}\n",
" if not path.startswith(('http://', 'https://')):\n",
" path = self.gh_host + path\n",
" if route:\n",
" for k,v in route.items(): route[k] = quote(str(route[k]))\n",
" return_json = ('json' in headers['Accept'])\n",
" return_json = ('json' in headers['Accept']) and (decode is True)\n",
" debug = self.debug if self.debug else print_summary if os.getenv('GHAPI_DEBUG') else None\n",
" res,self.recv_hdrs = urlsend(path, verb, headers=headers or None, debug=debug, return_headers=True,\n",
" res,self.recv_hdrs = urlsend(path, verb, headers=headers or None, decode=decode, debug=debug, return_headers=True,\n",
" route=route or None, query=query or None, data=data or None, return_json=return_json, timeout=timeout)\n",
" if 'X-RateLimit-Remaining' in self.recv_hdrs:\n",
" newlim = self.recv_hdrs['X-RateLimit-Remaining']\n",
Expand Down
21 changes: 16 additions & 5 deletions ghapi/core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,15 +36,26 @@ def _mk_sig(req_args, opt_args, anno_args):
params += [_mk_param(k, **_mk_sig_detls(v)) for k,v in anno_args.items()]
return Signature(params)

def _decode_response(path: str) -> bool:
"checks if a endpoint needs to have it's response from `fastcore.core.urlsend` decoded or just return json"
needs_decode = (
"/orgs/{org}/migrations/{migration_id}/archive",
"/repos/{owner}/{repo}/actions/artifacts/{artifact_id}/{archive_format}",
"/repos/{owner}/{repo}/tarball/{ref}",
"/repos/{owner}/{repo}/zipball/{ref}",
)
return path not in needs_decode

class _GhObj: pass

# %% ../00_core.ipynb 7
class _GhVerb(_GhObj):
__slots__ = 'path,verb,tag,name,summary,url,route_ps,params,data,preview,client,__doc__'.split(',')
__slots__ = 'path,verb,tag,name,summary,url,route_ps,params,data,preview,client,decode,__doc__'.split(',')
def __init__(self, path, verb, oper, summary, url, params, data, preview, client, kwargs):
tag,*name = oper.split('/')
name = '__'.join(name)
name = name.replace('-','_')
decode = _decode_response(path)
path,_,_ = partial_format(path, **kwargs)
route_ps = stringfmt_names(path)
__doc__ = summary
Expand All @@ -58,7 +69,7 @@ def __call__(self, *args, headers=None, **kwargs):
for a,b in zip(args,flds): kwargs[b]=a
route_p,query_p,data_p = [{p:kwargs[p] for p in o if p in kwargs}
for o in (self.route_ps,self.params,d)]
return self.client(self.path, self.verb, headers=headers, route=route_p, query=query_p, data=data_p)
return self.client(self.path, self.verb, headers=headers, decode=self.decode, route=route_p, query=query_p, data=data_p)

def __str__(self): return f'{self.tag}.{self.name}{signature(self)}\n{self.doc_url}'
@property
Expand Down Expand Up @@ -107,17 +118,17 @@ def __init__(self, owner=None, repo=None, token=None, jwt_token=None, debug=None
self.debug,self.limit_cb,self.limit_rem = debug,limit_cb,5000
self.gh_host = gh_host or GH_HOST

def __call__(self, path:str, verb:str=None, headers:dict=None, route:dict=None, query:dict=None, data=None, timeout=None):
def __call__(self, path:str, verb:str=None, headers:dict=None, route:dict=None, query:dict=None, data=None, timeout=None, decode=True):
"Call a fully specified `path` using HTTP `verb`, passing arguments to `fastcore.core.urlsend`"
if verb is None: verb = 'POST' if data else 'GET'
headers = {**self.headers,**(headers or {})}
if not path.startswith(('http://', 'https://')):
path = self.gh_host + path
if route:
for k,v in route.items(): route[k] = quote(str(route[k]))
return_json = ('json' in headers['Accept'])
return_json = ('json' in headers['Accept']) and (decode is True)
debug = self.debug if self.debug else print_summary if os.getenv('GHAPI_DEBUG') else None
res,self.recv_hdrs = urlsend(path, verb, headers=headers or None, debug=debug, return_headers=True,
res,self.recv_hdrs = urlsend(path, verb, headers=headers or None, decode=decode, debug=debug, return_headers=True,
route=route or None, query=query or None, data=data or None, return_json=return_json, timeout=timeout)
if 'X-RateLimit-Remaining' in self.recv_hdrs:
newlim = self.recv_hdrs['X-RateLimit-Remaining']
Expand Down
2 changes: 1 addition & 1 deletion settings.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ language = English
custom_sidebar = False
license = apache2
status = 5
requirements = fastcore>=1.7.0
requirements = fastcore>=1.7.2
dev_requirements = jsonref matplotlib
console_scripts = ghapi=ghapi.cli:ghapi
ghpath=ghapi.cli:ghpath
Expand Down

0 comments on commit 2093545

Please sign in to comment.