Skip to content

Commit

Permalink
add support for OGB datasets
Browse files Browse the repository at this point in the history
  • Loading branch information
@ChandlerBang
ChandlerBang committed Apr 13, 2021
1 parent 067f1e1 commit ee44178
Show file tree
Hide file tree
Showing 7 changed files with 54 additions and 14 deletions.
20 changes: 17 additions & 3 deletions deeprobust/graph/data/pyg_dataset.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,15 +117,23 @@ class Pyg2Dpr(Dataset):
"""

def __init__(self, pyg_data, **kwargs):
is_ogb = hasattr(pyg_data, 'get_idx_split')
if is_ogb: # get splits for ogb datasets
splits = pyg_data.get_idx_split()
pyg_data = pyg_data[0]
n = pyg_data.num_nodes
self.adj = sp.csr_matrix((np.ones(pyg_data.edge_index.shape[1]),
(pyg_data.edge_index[0], pyg_data.edge_index[1])), shape=(n, n))
self.features = pyg_data.x.numpy()
self.labels = pyg_data.y.numpy()
self.idx_train = mask_to_index(pyg_data.train_mask, n)
self.idx_val = mask_to_index(pyg_data.val_mask, n)
self.idx_test = mask_to_index(pyg_data.test_mask, n)
if is_ogb: # set splits for ogb datasets
self.idx_train = splits['train'].numpy()
self.idx_val = splits['valid'].numpy()
self.idx_test = splits['test'].numpy()
else:
self.idx_train = mask_to_index(pyg_data.train_mask, n)
self.idx_val = mask_to_index(pyg_data.val_mask, n)
self.idx_test = mask_to_index(pyg_data.test_mask, n)
self.name = 'Pyg2Dpr'

class AmazonPyg(Amazon):
Expand Down Expand Up @@ -276,3 +284,9 @@ def index_to_mask(index, size):
physics = CoauthorPyg(root='/tmp', name='physics')
print(physics)
print(physics[0])

# from ogb.nodeproppred import PygNodePropPredDataset
# dataset = PygNodePropPredDataset(name = 'ogbn-arxiv')
# ogb_data = Pyg2Dpr(dataset)


Binary file modified docs/_build/doctrees/environment.pickle
View file
Binary file not shown.
4 changes: 2 additions & 2 deletions docs/_build/html/_modules/deeprobust/image/netmodels/train_model.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,7 @@ <h1>Source code for deeprobust.image.netmodels.train_model</h1><div class="highl


<span class="n">optimizer</span> <span class="o">=</span> <span class="n">optim</span><span class="o">.</span><span class="n">SGD</span><span class="p">(</span><span class="n">train_net</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">lr</span><span class="o">=</span> <span class="mf">0.1</span><span class="p">,</span> <span class="n">momentum</span><span class="o">=</span><span class="mf">0.5</span><span class="p">)</span>
<span class="n">scheduler</span> <span class="o">=</span> <span class="n">lr_scheduler</span><span class="o">.</span><span class="n">StepLR</span><span class="p">(</span><span class="n">optimizer</span><span class="p">,</span> <span class="n">step_size</span> <span class="o">=</span> <span class="mi">100</span><span class="p">,</span> <span class="n">gamma</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">)</span>
<span class="n">scheduler</span> <span class="o">=</span> <span class="n">optim</span><span class="o">.</span><span class="n">lr_scheduler</span><span class="o">.</span><span class="n">StepLR</span><span class="p">(</span><span class="n">optimizer</span><span class="p">,</span> <span class="n">step_size</span> <span class="o">=</span> <span class="mi">100</span><span class="p">,</span> <span class="n">gamma</span> <span class="o">=</span> <span class="mf">0.1</span><span class="p">)</span>
<span class="n">save_model</span> <span class="o">=</span> <span class="kc">True</span>
<span class="k">for</span> <span class="n">epoch</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">maxepoch</span> <span class="o">+</span> <span class="mi">1</span><span class="p">):</span> <span class="c1">## 5 batches</span>

Expand Down Expand Up @@ -306,7 +306,7 @@ <h1>Source code for deeprobust.image.netmodels.train_model</h1><div class="highl
<span class="n">shuffle</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

<span class="n">test_loader</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">DataLoader</span><span class="p">(</span>
<span class="n">datasets</span><span class="o">.</span><span class="n">MNIST</span><span class="p">(</span><span class="s1">&#39;../data&#39;</span><span class="p">,</span> <span class="n">train</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">download</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
<span class="n">datasets</span><span class="o">.</span><span class="n">MNIST</span><span class="p">(</span><span class="n">data_dict</span><span class="p">,</span> <span class="n">train</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">download</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
<span class="n">transform</span><span class="o">=</span><span class="n">transforms</span><span class="o">.</span><span class="n">Compose</span><span class="p">([</span><span class="n">transforms</span><span class="o">.</span><span class="n">ToTensor</span><span class="p">(),</span>
<span class="n">transforms</span><span class="o">.</span><span class="n">Normalize</span><span class="p">((</span><span class="mf">0.1307</span><span class="p">,),</span> <span class="p">(</span><span class="mf">0.3081</span><span class="p">,))])),</span>
<span class="n">batch_size</span><span class="o">=</span><span class="mi">1000</span><span class="p">,</span>
Expand Down
15 changes: 7 additions & 8 deletions docs/_build/html/_modules/deeprobust/image/optimizer.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@
<h1>Source code for deeprobust.image.optimizer</h1><div class="highlight"><pre>
<span></span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">This module include the following optimizer:</span>
<span class="sd">1. differential_evolution: </span>
<span class="sd">1. differential_evolution:</span>
<span class="sd">The differential evolution global optimization algorithm</span>
<span class="sd">https://github.com/scipy/scipy/blob/70e61dee181de23fdd8d893eaa9491100e2218d7/scipy/optimize/_differentialevolution.py</span>

Expand All @@ -191,7 +191,6 @@ <h1>Source code for deeprobust.image.optimizer</h1><div class="highlight"><pre>
<span class="kn">from</span> <span class="nn">scipy.optimize</span> <span class="kn">import</span> <span class="n">OptimizeResult</span><span class="p">,</span> <span class="n">minimize</span>
<span class="kn">from</span> <span class="nn">scipy.optimize.optimize</span> <span class="kn">import</span> <span class="n">_status_message</span>
<span class="kn">from</span> <span class="nn">scipy._lib._util</span> <span class="kn">import</span> <span class="n">check_random_state</span>
<span class="kn">from</span> <span class="nn">scipy._lib.six</span> <span class="kn">import</span> <span class="n">xrange</span><span class="p">,</span> <span class="n">string_types</span>
<span class="kn">import</span> <span class="nn">warnings</span>


Expand Down Expand Up @@ -349,7 +348,7 @@ <h1>Source code for deeprobust.image.optimizer</h1><div class="highlight"><pre>
<span class="sd"> values. This has the effect of widening the search radius, but slowing</span>
<span class="sd"> convergence.</span>
<span class="sd"> .. versionadded:: 0.15.0</span>
<span class="sd"> </span>

<span class="sd"> References</span>
<span class="sd"> ----------</span>
<span class="sd"> .. [1] Storn, R and Price, K, Differential Evolution - a Simple and</span>
Expand Down Expand Up @@ -574,7 +573,7 @@ <h1>Source code for deeprobust.image.optimizer</h1><div class="highlight"><pre>
<span class="bp">self</span><span class="o">.</span><span class="n">parameter_count</span><span class="p">)</span>

<span class="bp">self</span><span class="o">.</span><span class="n">_nfev</span> <span class="o">=</span> <span class="mi">0</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">init</span><span class="p">,</span> <span class="n">string_types</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">init</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
<span class="k">if</span> <span class="n">init</span> <span class="o">==</span> <span class="s1">&#39;latinhypercube&#39;</span><span class="p">:</span>
<span class="bp">self</span><span class="o">.</span><span class="n">init_population_lhs</span><span class="p">()</span>
<span class="k">elif</span> <span class="n">init</span> <span class="o">==</span> <span class="s1">&#39;random&#39;</span><span class="p">:</span>
Expand Down Expand Up @@ -719,7 +718,7 @@ <h1>Source code for deeprobust.image.optimizer</h1><div class="highlight"><pre>
<span class="bp">self</span><span class="o">.</span><span class="n">_calculate_population_energies</span><span class="p">()</span>

<span class="c1"># do the optimisation.</span>
<span class="k">for</span> <span class="n">nit</span> <span class="ow">in</span> <span class="n">xrange</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">maxiter</span> <span class="o">+</span> <span class="mi">1</span><span class="p">):</span>
<span class="k">for</span> <span class="n">nit</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">maxiter</span> <span class="o">+</span> <span class="mi">1</span><span class="p">):</span>
<span class="c1"># evolve the population by a generation</span>
<span class="k">try</span><span class="p">:</span>
<span class="nb">next</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
Expand Down Expand Up @@ -812,7 +811,7 @@ <h1>Source code for deeprobust.image.optimizer</h1><div class="highlight"><pre>
<span class="c1">##############</span>
<span class="c1">##############</span>



<span class="n">minval</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">argmin</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">population_energies</span><span class="p">)</span>

Expand Down Expand Up @@ -1005,7 +1004,7 @@ <h1>Source code for deeprobust.image.optimizer</h1><div class="highlight"><pre>
<span class="sd"> currenttobest1bin, currenttobest1exp</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">r0</span><span class="p">,</span> <span class="n">r1</span> <span class="o">=</span> <span class="n">samples</span><span class="p">[:</span><span class="mi">2</span><span class="p">]</span>
<span class="n">bprime</span> <span class="o">=</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">population</span><span class="p">[</span><span class="n">candidate</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale</span> <span class="o">*</span>
<span class="n">bprime</span> <span class="o">=</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">population</span><span class="p">[</span><span class="n">candidate</span><span class="p">]</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale</span> <span class="o">*</span>
<span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">population</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">population</span><span class="p">[</span><span class="n">candidate</span><span class="p">]</span> <span class="o">+</span>
<span class="bp">self</span><span class="o">.</span><span class="n">population</span><span class="p">[</span><span class="n">r0</span><span class="p">]</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">population</span><span class="p">[</span><span class="n">r1</span><span class="p">]))</span>
<span class="k">return</span> <span class="n">bprime</span>
Expand Down Expand Up @@ -1045,7 +1044,7 @@ <h1>Source code for deeprobust.image.optimizer</h1><div class="highlight"><pre>

<div class="viewcode-block" id="AdamOptimizer"><a class="viewcode-back" href="../../../source/deeprobust.image.html#deeprobust.image.optimizer.AdamOptimizer">[docs]</a><span class="k">class</span> <span class="nc">AdamOptimizer</span><span class="p">:</span>
<span class="sd">&quot;&quot;&quot;Basic Adam optimizer implementation that can minimize w.r.t.</span>
<span class="sd"> a single variable. </span>
<span class="sd"> a single variable.</span>
<span class="sd"> Parameters</span>
<span class="sd"> ----------</span>
<span class="sd"> shape : tuple</span>
Expand Down
2 changes: 1 addition & 1 deletion docs/_build/html/searchindex.js
View file

Large diffs are not rendered by default.

14 changes: 14 additions & 0 deletions docs/graph/data.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,20 @@ For example, we can first create an instance of the Dataset class and convert it
dpr_data = Pyg2Dpr(pyg_data) # convert pyg to dpr
print(dpr_data.adj)
Load OGB Datasets
-----------------------
`Open Graph Benchmark (OGB) <https://ogb.stanford.edu/>`_ has provided various benchmark
datasets. DeepRobsut now provides interface to convert OGB dataset format (Pyg data format)
to DeepRobust format.

.. code-block:: python
from ogb.nodeproppred import PygNodePropPredDataset
pyg_data = PygNodePropPredDataset(name = 'ogbn-arxiv')
dpr_data = Pyg2Dpr(pyg_data) # convert pyg to dpr
Load Pytorch Geometric Amazon and Coauthor Datasets
-----------------------
DeepRobust also provides access to the Amazon datasets and Coauthor datasets, i.e.,
Expand Down
13 changes: 13 additions & 0 deletions docs/graph/pyg.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,19 @@ then update its :obj:`edge_index`:
Now :obj:`pyg_data` becomes the perturbed data in the format of PyTorch Geometric.
We can then use it as the input for various Pytorch Geometric models!

Load OGB Datasets
-----------------------
`Open Graph Benchmark (OGB) <https://ogb.stanford.edu/>`_ has provided various benchmark
datasets. DeepRobsut now provides interface to convert OGB dataset format (Pyg data format)
to DeepRobust format.

.. code-block:: python
from ogb.nodeproppred import PygNodePropPredDataset
pyg_data = PygNodePropPredDataset(name = 'ogbn-arxiv')
dpr_data = Pyg2Dpr(pyg_data) # convert pyg to dpr
Load Pytorch Geometric Amazon and Coauthor Datasets
-----------------------
DeepRobust also provides access to the Amazon datasets and Coauthor datasets, i.e.,
Expand Down

0 comments on commit ee44178

Please sign in to comment.