Skip to content

Commit

Permalink
feat: Add support for arrayful JSON (#1647)
Browse files Browse the repository at this point in the history 
* Allow for tensor-like values in model spec by building the
modifier builder_data from a list of lists/arrays and then
using tensorlib.concatenate to built up the sample properties.
* Add test for building a model with tensor inputs.
  • Loading branch information
@lukasheinrich
lukasheinrich authored Oct 18, 2021
1 parent fb3a727 commit be5ae65
Show file tree
Hide file tree
Showing 5 changed files with 177 additions and 14 deletions.
26 changes: 20 additions & 6 deletions src/pyhf/modifiers/histosys.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import logging

from pyhf import get_backend, events
from pyhf import get_backend, default_backend, events
from pyhf import interpolators
from pyhf.parameters import ParamViewer

Expand Down Expand Up @@ -31,7 +31,7 @@ def __init__(self, config):
def collect(self, thismod, nom):
lo_data = thismod['data']['lo_data'] if thismod else nom
hi_data = thismod['data']['hi_data'] if thismod else nom
maskval = True if thismod else False
maskval = bool(thismod)
mask = [maskval] * len(nom)
return {'lo_data': lo_data, 'hi_data': hi_data, 'mask': mask, 'nom_data': nom}

Expand All @@ -45,10 +45,10 @@ def append(self, key, channel, sample, thismod, defined_samp):
else [0.0] * self.config.channel_nbins[channel]
)
moddata = self.collect(thismod, nom)
self.builder_data[key][sample]['data']['lo_data'] += moddata['lo_data']
self.builder_data[key][sample]['data']['hi_data'] += moddata['hi_data']
self.builder_data[key][sample]['data']['nom_data'] += moddata['nom_data']
self.builder_data[key][sample]['data']['mask'] += moddata['mask']
self.builder_data[key][sample]['data']['lo_data'].append(moddata['lo_data'])
self.builder_data[key][sample]['data']['hi_data'].append(moddata['hi_data'])
self.builder_data[key][sample]['data']['nom_data'].append(moddata['nom_data'])
self.builder_data[key][sample]['data']['mask'].append(moddata['mask'])

if thismod:
self.required_parsets.setdefault(
Expand All @@ -57,6 +57,20 @@ def append(self, key, channel, sample, thismod, defined_samp):
)

def finalize(self):
for modifier in self.builder_data.values():
for sample in modifier.values():
sample["data"]["mask"] = default_backend.concatenate(
sample["data"]["mask"]
)
sample["data"]["lo_data"] = default_backend.concatenate(
sample["data"]["lo_data"]
)
sample["data"]["hi_data"] = default_backend.concatenate(
sample["data"]["hi_data"]
)
sample["data"]["nom_data"] = default_backend.concatenate(
sample["data"]["nom_data"]
)
return self.builder_data


Expand Down
17 changes: 14 additions & 3 deletions src/pyhf/modifiers/shapesys.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@ def append(self, key, channel, sample, thismod, defined_samp):
else [0.0] * self.config.channel_nbins[channel]
)
moddata = self.collect(thismod, nom)
self.builder_data[key][sample]['data']['mask'] += moddata['mask']
self.builder_data[key][sample]['data']['uncrt'] += moddata['uncrt']
self.builder_data[key][sample]['data']['nom_data'] += moddata['nom_data']
self.builder_data[key][sample]['data']['mask'].append(moddata['mask'])
self.builder_data[key][sample]['data']['uncrt'].append(moddata['uncrt'])
self.builder_data[key][sample]['data']['nom_data'].append(moddata['nom_data'])

if thismod:
self.required_parsets.setdefault(
Expand All @@ -62,6 +62,17 @@ def append(self, key, channel, sample, thismod, defined_samp):
)

def finalize(self):
for modifier in self.builder_data.values():
for sample in modifier.values():
sample["data"]["mask"] = default_backend.concatenate(
sample["data"]["mask"]
)
sample["data"]["uncrt"] = default_backend.concatenate(
sample["data"]["uncrt"]
)
sample["data"]["nom_data"] = default_backend.concatenate(
sample["data"]["nom_data"]
)
return self.builder_data


Expand Down
17 changes: 14 additions & 3 deletions src/pyhf/modifiers/staterror.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,9 @@ def append(self, key, channel, sample, thismod, defined_samp):
else [0.0] * self.config.channel_nbins[channel]
)
moddata = self.collect(thismod, nom)
self.builder_data[key][sample]['data']['mask'] += moddata['mask']
self.builder_data[key][sample]['data']['uncrt'] += moddata['uncrt']
self.builder_data[key][sample]['data']['nom_data'] += moddata['nom_data']
self.builder_data[key][sample]['data']['mask'].append(moddata['mask'])
self.builder_data[key][sample]['data']['uncrt'].append(moddata['uncrt'])
self.builder_data[key][sample]['data']['nom_data'].append(moddata['nom_data'])

if thismod:
self.required_parsets.setdefault(
Expand All @@ -53,6 +53,17 @@ def append(self, key, channel, sample, thismod, defined_samp):
)

def finalize(self):
for modifier in self.builder_data.values():
for sample in modifier.values():
sample["data"]["mask"] = default_backend.concatenate(
sample["data"]["mask"]
)
sample["data"]["uncrt"] = default_backend.concatenate(
sample["data"]["uncrt"]
)
sample["data"]["nom_data"] = default_backend.concatenate(
sample["data"]["nom_data"]
)
return self.builder_data


Expand Down
7 changes: 5 additions & 2 deletions src/pyhf/pdf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,11 +61,14 @@ def append(self, channel, sample, defined_samp):
raise exceptions.InvalidModel(
f'expected {self.config.channel_nbins[channel]} size sample data but got {len(nom)}'
)
self.mega_samples[sample]['nom'] += nom
self.mega_samples[sample]['nom'].append(nom)

def finalize(self):
nominal_rates = default_backend.astensor(
[self.mega_samples[sample]['nom'] for sample in self.config.samples]
[
default_backend.concatenate(self.mega_samples[sample]['nom'])
for sample in self.config.samples
]
)
_nominal_rates = default_backend.reshape(
nominal_rates,
Expand Down
124 changes: 124 additions & 0 deletions tests/test_pdf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -955,3 +955,127 @@ def test_par_names_scalar_nonscalar():
'scalar',
'nonscalar[0]',
]


def test_make_model_with_tensors():
def make_model(
nominal,
lumi_sigma,
corrup_data,
corrdn_data,
stater_data,
normsys_up,
normsys_dn,
uncorr_data,
):
spec = {
"channels": [
{
"name": "achannel",
"samples": [
{
"name": "background",
"data": nominal,
"modifiers": [
{"name": "mu", "type": "normfactor", "data": None},
{"name": "lumi", "type": "lumi", "data": None},
{
"name": "mod_name",
"type": "shapefactor",
"data": None,
},
{
"name": "corr_bkguncrt2",
"type": "histosys",
"data": {
'hi_data': corrup_data,
'lo_data': corrdn_data,
},
},
{
"name": "staterror2",
"type": "staterror",
"data": stater_data,
},
{
"name": "norm",
"type": "normsys",
"data": {'hi': normsys_up, 'lo': normsys_dn},
},
],
}
],
},
{
"name": "secondchannel",
"samples": [
{
"name": "background",
"data": nominal,
"modifiers": [
{"name": "mu", "type": "normfactor", "data": None},
{"name": "lumi", "type": "lumi", "data": None},
{
"name": "mod_name",
"type": "shapefactor",
"data": None,
},
{
"name": "uncorr_bkguncrt2",
"type": "shapesys",
"data": uncorr_data,
},
{
"name": "corr_bkguncrt2",
"type": "histosys",
"data": {
'hi_data': corrup_data,
'lo_data': corrdn_data,
},
},
{
"name": "staterror",
"type": "staterror",
"data": stater_data,
},
{
"name": "norm",
"type": "normsys",
"data": {'hi': normsys_up, 'lo': normsys_dn},
},
],
}
],
},
],
}
model = pyhf.Model(
{
'channels': spec['channels'],
'parameters': [
{
'name': 'lumi',
'auxdata': [1.0],
'bounds': [[0.5, 1.5]],
'inits': [1.0],
"sigmas": [lumi_sigma],
}
],
},
validate=False,
)

pars = model.config.suggested_init()
exp_data = model.expected_data(pars)
assert exp_data is not None

make_model(
pyhf.tensorlib.astensor([60.0, 62.0]),
pyhf.tensorlib.astensor(0.2),
pyhf.tensorlib.astensor([60.0, 62.0]),
pyhf.tensorlib.astensor([60.0, 62.0]),
pyhf.tensorlib.astensor([5.0, 5.0]),
pyhf.tensorlib.astensor(0.95),
pyhf.tensorlib.astensor(1.05),
pyhf.tensorlib.astensor([5.0, 5.0]),
)

0 comments on commit be5ae65

Please sign in to comment.