Skip to content

Commit

Permalink
Cmor 3.2.3 (#159)
Browse files Browse the repository at this point in the history 
* add travis CI for Pushes

* add service docker to travis

* change cdms to cmor and comment out test

* add script

* It would help to acually have the scripts

* change to PCMDI channel

* try with osx

* error in PCMDI token

* Fix osx miniconda link

* add entry_point

* update tables and PrePARE

* fix test

* add parent experiment id and sub experiment id validation

* Fix some tests

* Fix all Tables for tests

* allow a maximum of 30 tables to be loaded in memory

* Add Test/test_python_toomany_tables.py

* Fix PrePARE entry_points using setuptools

* Check filename for for PrePARE

* Fix member_id and variant_label confusion

* add numpy 1.12 to anaconda upload

* update tables and CMIP6_CV

* Fix PrePARE filname check

* fix log problem

* clean code

* fix Grid pywrapper issues and clean code

* add test to Makefile.in and fix test_interface_03 directory

* create source in CMIP6_CV.json file for sourc_id section

* autopep8 files and increase to 3.2.3

* fix .travis.yaml
  • Loading branch information
@dnadeau4
dnadeau4 authored Apr 19, 2017
1 parent fa4bbd0 commit 7406094
Show file tree
Hide file tree
Showing 206 changed files with 16,922 additions and 12,598 deletions.
58 changes: 30 additions & 28 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,18 @@
os:
- linux
- osx

python:
- 2.7
#sudo: required
services:
- docker

sudo: false

before_install:
# - if [ "$TRAVIS_OS_NAME" = "linux" ]; then sudo apt-get update -qq; sudo apt-get install -y gfortran ; fi
- if [ "$TRAVIS_OS_NAME" = "linux" ]; then wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh; fi

# - if [ "$TRAVIS_OS_NAME" = "osx" ]; then brew update ; fi
- if [ "$TRAVIS_OS_NAME" = "osx" ]; then wget https://repo.continuum.io/miniconda/Miniconda-latest-MacOSX-x86_64.sh -O miniconda.sh; fi
# - if [ "$TRAVIS_OS_NAME" = "osx" ]; then wget https://github.com/UV-CDAT/uvcdat/releases/download/v2.4.1/gfortran-4.9.2-Mac.tar.gz -O ~/gfortran-4.9.2-Mac.tar.gz ; pushd / ; sudo tar xzvf ~/gfortran-4.9.2-Mac.tar.gz ; popd ; fi
- if [ "$TRAVIS_OS_NAME" = "linux" ]; then wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh
-O miniconda.sh; fi
- if [ "$TRAVIS_OS_NAME" = "osx" ]; then wget https://repo.continuum.io/miniconda/Miniconda2-latest-MacOSX-x86_64.sh
-O miniconda.sh; fi
- export PATH="$HOME/miniconda/bin:$PATH"
- export PREFIX="$HOME/miniconda"
- export UDUNITS2_XML_PATH=$HOME/miniconda/share/udunits/udunits2.xml
Expand All @@ -24,28 +23,31 @@ before_install:
- conda install openssl=1.0.2d
- conda install cdms2 -c uvcdat/label/nightly -c conda-forge -c uvcdat
- conda install udunits2 -c uvcdat/label/nightly -c conda-forge -c uvcdat
- conda install gcc
- conda install nose
- conda install gcc

install:
- git clone https://github.com/PCMDI/cmip6-cmor-tables
- ln -s cmip6-cmor-tables/Tables Tables
- ./configure --prefix=$PREFIX --with-python --with-uuid=$PREFIX --with-udunits2=$PREFIX --with-netcdf=$PREFIX --enable-verbose-test
- make install
- git clone https://github.com/PCMDI/cmip6-cmor-tables
- ln -s cmip6-cmor-tables/Tables Tables
- "./configure --prefix=$PREFIX --with-python --with-uuid=$PREFIX --with-udunits2=$PREFIX
--with-netcdf=$PREFIX --enable-verbose-test"
- make install

script:
- make test
- for file in `ls -1 Test/test_python_CMIP6_CV*.py`; do echo $file; python $file; if [[ "$?" != "0" ]]; then return $?; fi; done
# - nosetests -vv --collect-only Test/test_python_CMIP6_CV_*.py
# - nosetests -vv --collect-only Test/test_python_has_*.py
# - for file in `ls -1 Test/test_python_CMIP6_CV*.py`; do echo $file; python $file; done
# - python run_tests2.py
# - python run_tests.py
script:
- make test
- for file in `ls -1 Test/test_python_CMIP6_CV*.py`; do echo $file; python $file; mystatus=$?; if [[ "$mystatus" != "0" ]]; then return ${mystatus}; fi; done

after_failure:
- ulimit -a
- free -m
- find CMIP6/
- df -kh ./
- ls -lR
- cat ~/build.sh
- ulimit -a
- free -m
- find CMIP6/
- df -kh ./
- ls -lR
- cat ~/build.sh

env:
global:
secure: V5q+wzxrJ5Rmk1vVYXimIdHgWUwZSHINd5W+oi3mTmfbtVyrngau9XQVZyiPyg8S309vHAbys9uMaplp83to7OY1Cw3+w002oMWkCaLwlVMn4UWs+xU7lJGn/wwLwN1OikZnTtiXF8RqrtBZpYsbAxjdebqZ8uZx/zDJXHVvvZY=

after_success:
- if [ "$TRAVIS_OS_NAME" = "osx" ]; then bash scripts/conda_upload.sh ${TRAVIS_BRANCH} ; fi
- if [ "$TRAVIS_OS_NAME" = "linux" ]; then docker run -v `pwd`:/travis_home -e CONDA_UPLOAD_TOKEN=${CONDA_UPLOAD_TOKEN} -a STDOUT -a STDERR cdat/conda:conda-forge-cdms2 /travis_home/scripts/conda_upload.sh ${TRAVIS_BRANCH} ; fi
4 changes: 2 additions & 2 deletions INSTALL
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
INSTALLATION INSTRUCTIONS
-------------------------
Climate Model Output Rewriter (CMOR) version 3.2.2 installation instructions.
Climate Model Output Rewriter (CMOR) version 3.2.3 installation instructions.


DOWNLOAD
Expand All @@ -13,7 +13,7 @@ cd CMOR

INSTALLATION
------------
CMOR 3.2.2 requires external packages that need to be installed first.
CMOR 3.2.3 requires external packages that need to be installed first.
It can be compiled/linked against either NetCDF3 or NetCDF4. If you
decide to go with NetCDF4 be sure to build NetCDF4 with the
--enable-netcdf-4 option!
Expand Down
6 changes: 3 additions & 3 deletions Lib/pywrapper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,11 +197,11 @@ def set_grid_mapping(grid_id,mapping_name,parameter_names,parameter_values=None,
raise Exception, "mapping name must be a string"

if isinstance(parameter_names,dict):
pnams = []
pnms = []
pvals = []
punit = []
for k in parameter_names.keys():
pnams.append(k)
pnms.append(k)
val = parameter_names[k]
if isinstance(val,dict):
ks = val.keys()
Expand All @@ -219,7 +219,7 @@ def set_grid_mapping(grid_id,mapping_name,parameter_names,parameter_values=None,
pvals.append(float(v))
except:
pass
if len(pvals)!=len(punits) or len(pvals)!=len(pnams):
if len(pvals)!=len(punit) or len(pvals)!=len(pnms):
raise Exception, "could not figure out values for parameter_name: '%s' " % k
else:
raise Exception, "could not figure out values for parameter_name: '%s' " % k
Expand Down
75 changes: 70 additions & 5 deletions LibCV/PrePARE/PrePARE.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
@author: Denis Nadeau LLNL
'''

import cmip6_cv
import cdms2
import argparse
Expand Down Expand Up @@ -130,9 +129,9 @@ def __init__(self, args):
# -------------------------------------------------------------------
# find variable that contains a "history" (should only be one)
# -------------------------------------------------------------------
self.var = [var for var in self.variables if 'history' in self.infile.listattribute(var)]
self.var = [self.infile.variable_id]

if((self.var == []) or (len(self.var) > 1)):
if((self.var == []) or (len(self.var) > 1)):
print bcolors.FAIL
print "!!!!!!!!!!!!!!!!!!!!!!!!!"
print "! Error: The input file does not have an history attribute and the CMIP6 variable could not be found"
Expand Down Expand Up @@ -171,25 +170,43 @@ def __init__(self, args):
cmip6_cv.set_cur_dataset_attribute(cmip6_cv.CMOR_AXIS_ENTRY_FILE, "CMIP6_coordinate.json")
cmip6_cv.set_cur_dataset_attribute(cmip6_cv.CMOR_FORMULA_VAR_FILE, "CMIP6_formula_terms.json")


# -------------------------------------------------------------------
# Create alist of all Global Attributes and set "dataset"
# -------------------------------------------------------------------
self.dictGbl = {key: self.infile.getglobal(key) for key in self.attributes}
ierr = [cmip6_cv.set_cur_dataset_attribute(key, value) for key, value in self.dictGbl.iteritems()]
if(self.dictGbl["sub_experiment_id"] not in ["none"]):
member_id = self.dictGbl["sub_experiment_id"] + '-' +self.dictGbl["variant_label"]
else:
member_id = self.dictGbl["variant_label"]
cmip6_cv.set_cur_dataset_attribute(cmip6_cv.GLOBAL_ATT_MEMBER_ID, member_id)

self.setDoubleValue('branch_time_in_parent')
self.setDoubleValue('branch_time_in_child')
# -------------------------------------------------------------------
# Create a dictionnary of attributes for var
# -------------------------------------------------------------------
self.dictVars = dict((y, x) for y, x in
[(key, value) for key in self.keys
if self.infile.getattribute(self.var[0], key) is not None
for value in [self.infile.getattribute(self.var[0], key)]])
try:
self.calendar = self.infile.getAxis('time').calendar
self.timeunits = self.infile.getAxis('time').units
except:
self.calendar = "gregorian"
self.timeunits = "days since ?"
# -------------------------------------------------------------------
# Load CMIP6 table into memory
# -------------------------------------------------------------------
self.table_id = cmip6_cv.load_table(self.cmip6_table)

def setDoubleValue(self, attribute):
if( cmip6_cv.has_cur_dataset_attribute(attribute) ):
if(isinstance(self.dictGbl[attribute],numpy.ndarray) and type(self.dictGbl[attribute][0]) == numpy.float64):
self.dictGbl[attribute] = self.dictGbl[attribute][0]
cmip6_cv.set_cur_dataset_attribute(attribute,self.dictGbl[attribute])

def ControlVocab(self):
'''
Check CMIP6 global attributes against Control Vocabulary file.
Expand All @@ -210,7 +227,55 @@ def ControlVocab(self):
cmip6_cv.check_grids(self.table_id)
cmip6_cv.check_ISOTime()
cmip6_cv.check_furtherinfourl(self.table_id)
varid = cmip6_cv.setup_variable(self.var[0], 'm', 1e20)
cmip6_cv.check_parentExpID(self.table_id)
cmip6_cv.check_subExpID(self.table_id)
try:
startimebnds = self.infile['time_bnds'][0][0]
endtimebnds = self.infile['time_bnds'][-1][1]
except:
startimebnds = 0
endtimebnds = 0
try:
startime = self.infile['time'][0]
endtime = self.infile['time'][-1]
except:
startime = 0
endtime = 0
varunits = self.infile[self.var[0]].units
varmissing = self.infile[self.var[0]]._FillValue[0]
varid = cmip6_cv.setup_variable(self.var[0], varunits, varmissing, startime, endtime,
startimebnds, endtimebnds)
fn = os.path.basename(self.infile.id)
cmip6_cv.check_filename(self.table_id, varid, self.calendar, self.timeunits, fn)

if not isinstance(self.dictGbl['realization_index'], numpy.ndarray):
print bcolors.FAIL
print "====================================================================================="
print "realization_index is not an integer: ", type(self.dictGbl['realization_index'])
print "====================================================================================="
print bcolors.ENDC
cmip6_cv.set_CV_Error()
if not isinstance(self.dictGbl['initialization_index'], numpy.ndarray):
print bcolors.FAIL
print "====================================================================================="
print "initialization_index is not an integer: ", type(self.dictGbl['initialization_index'])
print "====================================================================================="
print bcolors.ENDC
cmip6_cv.set_CV_Error()
if not isinstance(self.dictGbl['physics_index'], numpy.ndarray):
print bcolors.FAIL
print "====================================================================================="
print "physics_index is not an integer: ", type(self.dictGbl['physics_index'])
print "====================================================================================="
print bcolors.ENDC
cmip6_cv.set_CV_Error()
if not isinstance(self.dictGbl['forcing_index'], numpy.ndarray):
print bcolors.FAIL
print "====================================================================================="
print "forcing_index is not an integer: ", type(self.dictGbl['forcing_index'])
print "====================================================================================="
print bcolors.ENDC
cmip6_cv.set_CV_Error()

prepLIST = cmip6_cv.list_variable_attributes(varid)
for key in prepLIST:
Expand Down
1 change: 1 addition & 0 deletions LibCV/PrePARE/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
from PrePARE import main
2 changes: 1 addition & 1 deletion LibCV/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from cmor_const import *

from pywrapper import setup, load_table, set_table, close, set_cur_dataset_attribute, get_cur_dataset_attribute, has_cur_dataset_attribute, set_variable_attribute, list_variable_attributes, get_variable_attribute, has_variable_attribute, check_institution, check_sourceID, check_experiment, check_grids, check_requiredattributes, check_ISOTime, check_furtherinfourl, get_CV_Error, setup_variable, set_CV_Error
from pywrapper import setup, load_table, set_table, close, set_cur_dataset_attribute, get_cur_dataset_attribute, has_cur_dataset_attribute, set_variable_attribute, list_variable_attributes, get_variable_attribute, has_variable_attribute, check_institution, check_sourceID, check_experiment, check_grids, check_requiredattributes, check_ISOTime, check_furtherinfourl, get_CV_Error, setup_variable, set_CV_Error, check_parentExpID, check_subExpID, check_filename

1 change: 1 addition & 0 deletions LibCV/cmor_const.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
FILE_PATH_TEMPLATE
FILE_NAME_TEMPLATE
GLOBAL_ATT_FURTHERINFOURLTMPL
GLOBAL_ATT_MEMBER_ID
CMOR_AXIS_ENTRY_FILE
CMOR_FORMULA_VAR_FILE
"""
Expand Down
56 changes: 53 additions & 3 deletions LibCV/pywrapper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ def set_cur_dataset_attribute(name, value):
def has_cur_dataset_attribute(name):
"""determines if the current cmor dataset has an attribute
Usage:
cmor.het_cur_dataset_attribute(name)
cmor.has_cur_dataset_attribute(name)
Where:
name: is the name of the attribute
Returns True if the dataset has the attribute, False otherwise
Expand Down Expand Up @@ -256,6 +256,21 @@ def check_sourceID(table_id):
return(ierr)


def check_filename(table_id, var_name, calendar, timeunits, infile):
'''
Validate filename with timestamp for current variable and file
Usage:
cmip6_cv.check_filename(table_id, var_id)
Where:
table_id is the table id returned by load_table()
var_name is the variable name
Return 0 on success
'''
ierr = _cmip6_cv.check_filename(table_id, var_name,
calendar, timeunits, infile)
return(ierr)

def check_experiment(table_id):
'''
Validate Experiment and Experiement_id against Control Vocabulary file.
Expand Down Expand Up @@ -300,6 +315,37 @@ def check_requiredattributes(table_id):
return(ierr)


def check_subExpID(table_id):
'''
Validate that sub_experiment ind sub_experiment_id are set to appropriate value
as defined in the CV file.
Usage:
cmip6_cv.check_subExpID(table_id)
Where:
table_id is the table id returned by load_table()
Return 0 on success
'''
ierr = _cmip6_cv.check_subExpID(table_id)
return(ierr)

def check_parentExpID(table_id):
'''
Validate that parent_experiement is set to appropriate value
if parent is set to "no parent" validate that other related
attributes are set to "no parent"
if parent is set to any string, validate all related attributes.
Usage:
cmip6_cv.check_parentExpID(table_id)
Where:
table_id is the table id returned by load_table()
Return 0 on success
'''
ierr = _cmip6_cv.check_parentExpID(table_id)
return(ierr)

def check_ISOTime():
'''
Validate that creation attribute contains time
Expand All @@ -313,7 +359,7 @@ def check_ISOTime():
return(ierr)


def setup_variable(name, units, missing):
def setup_variable(name, units, missing, startime, endtime, startimebnds, endtimebnds):
'''
Create variable attributes from the table loaded by load_table.
Expand All @@ -323,10 +369,14 @@ def setup_variable(name, units, missing):
name is the variable name to validate
units are the variable units
missing is the missing value for this variable.
startime: time value for first part of timestap -- time[0]
endtime: time value for last part of timestap -- time[-1]
startimebnds: time bound value for first part of timestap -- time_bnds[0]
endtimebnds: time bound value for last part of timestap -- time_bnds[-1]
return: variable_id on success
-1 on failure
'''
ierr = _cmip6_cv.setup_variable(name, units, missing)
ierr = _cmip6_cv.setup_variable(name, units, missing, startime, endtime, startimebnds, endtimebnds)
return(ierr)
def get_CV_Error():
'''
Expand Down
2 changes: 2 additions & 0 deletions Makefile.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,8 @@ test_a_python:
@echo "${OK_COLOR}Testing ${TEST_NAME} ${NO_COLOR}"
${PYTHONEXEC} ${TEST_NAME} ${VERB}
test_python: python
env TEST_NAME=Test/test_python_CMIP6_projections.py make test_a_python
env TEST_NAME=Test/test_python_toomany_tables.py make test_a_python
env TEST_NAME=Test/test_chunking.py make test_a_python
env TEST_NAME=Test/test_python_direct_calls.py make test_a_python
env TEST_NAME=Test/test_python_user_interface_00.py make test_a_python
Expand Down
Loading

0 comments on commit 7406094

Please sign in to comment.